Pytest precommit mark removal

Author: sgonorov

Jenkinsfile

@@ -8,9 +8,6 @@ properties([
         booleanParam(defaultValue: true,
                      description: 'Whether to propagate commit status to GitHub',
                      name: 'propagateStatus'),
-        booleanParam(defaultValue: false,
-               description: 'If true, forces running pre-commit scope',
-               name: 'forceRunPrecommitScope'),
         string(defaultValue: '',
                description: 'Pipeline shared library version (branch/tag/commit). Determined automatically if empty',
                name: 'library_version')

pyproject.toml

@@ -59,8 +59,3 @@ requires = [
     "cmake~=3.24.0; platform_system == 'Darwin' and platform_machine == 'arm64'",
 ]
 build-backend = "py_build_cmake.build"
-
-[tool.pytest.ini_options]
-markers = [
-    "precommit: (deselect with '-m \"precommit\"')",
-]

src/cpp/src/lora/adapter.cpp

@@ -100,10 +100,9 @@ struct AutoSafetensor: public safetensors_File {
 ConstantMap safetensor_to_constant_map(const ov::Tensor& safetensor) {
     AutoSafetensor safe_tensors_file{};
 
-    // Intentionally discard constness as safetensors_file_init requires a non-const pointer (used as read-only)
-    auto data_ptr = const_cast<char*>(safetensor.data<char>());
-    OPENVINO_ASSERT(safetensors_file_init(data_ptr, safetensor.get_byte_size(), &safe_tensors_file) == nullptr,
-                    "Cannot parse safetensor as a Safetensors file format. Safetensors file format is supported only");
+    OPENVINO_ASSERT(safetensors_file_init(safetensor.data<char>(), safetensor.get_byte_size(), &safe_tensors_file) == nullptr,
+        "Cannot parse safetensor as a Safetensors file format. Safetensors file format is supported only"
+    );
 
     ConstantMap tensors;
     for (int i = 0; i < safe_tensors_file.num_tensors; i++) {

src/cpp/src/speculative_decoding/speculative_decoding_stateful.cpp

@@ -338,7 +338,7 @@ std::variant<int64_t, std::vector<int64_t>>
 
     auto sample_token = [&](const ov::Tensor& logits, std::size_t idx) {
         size_t sequence_offset = idx * vocab_size;
-        const float* logits_data = logits.data<float>() + sequence_offset;
+        float* logits_data = logits.data<float>() + sequence_offset;
         return std::max_element(logits_data, logits_data + vocab_size) - logits_data;
     };
 

tests/python_tests/README.md

@@ -14,29 +14,29 @@ pip install -r tests/python_tests/requirements.txt
 ## Run Tests
 
 ```sh
-python -m pytest tests/python_tests/ -m precommit
+python -m pytest tests/python_tests/
 ```
 
 If you have built GenAI library by yourself instead of using wheel please set `PYTHONPATH` so that test could find library, e.g.
 ```sh
-PYTHONPATH=$PYTHONPATH:.../openvino.genai/build-Release/ python -m pytest tests/python_tests/ -m precommit
+PYTHONPATH=$PYTHONPATH:.../openvino.genai/build-Release/ python -m pytest tests/python_tests/
 ```
 
 ## Customize tests run
 
-Tests have `precommit` set of models. `precommit` contains lightweight models which can be quickly inferred. If you wish to run specific tests, you can use `-k` option, for example to run only multibatch and chat tests:
+Tests have different sets of models for different purposes. If you wish to run specific tests, you can use `-k` option, for example to run only multibatch and chat tests:
 ```sh
-python -m pytest tests/python_tests/ -m precommit -k "test_multibatch and test_chat"
+python -m pytest tests/python_tests/ -k "test_multibatch and test_chat"
 ```
 
 If you wish to run all tests except beam search do the following:
 ```sh
-python -m pytest tests/python_tests/ -m precommit -k "not test_beam_search"
+python -m pytest tests/python_tests/ -k "not test_beam_search"
 ```
 
 Argument `--model_ids` can be used to run tests selectively only for specific models. HF model ids should be separated by space, e.g:
 ```sh
-python -m pytest tests/python_tests/ -m precommit -k "test_multibatch" --model_ids "TinyLlama/TinyLlama-1.1B-Chat-v1.0 Qwen/Qwen2-0.5B-Instruct"
+python -m pytest tests/python_tests/ -k "test_multibatch" --model_ids "TinyLlama/TinyLlama-1.1B-Chat-v1.0 Qwen/Qwen2-0.5B-Instruct"
 ```
 
 List of currently supported models can be found in tests/python_tests/models.py:get_models_list

tests/python_tests/conftest.py

@@ -53,6 +53,4 @@ def pytest_addoption(parser):
 
 
 def pytest_configure(config: pytest.Config):
-    marker = "precommit" if config.getoption("-m") == "precommit" else None
-    pytest.run_marker = marker
     pytest.selected_model_ids = config.getoption("--model_ids", default=None)

tests/python_tests/models/precommit renamed to tests/python_tests/models/lightweight

@@ -2,8 +2,8 @@
 
 markers =
     ; The following markers are defined for categorizing tests:
-    ; precommit   - Tests that should be run before committing code.
     ; real_models - Tests that involve execution of the models from models/real_models file
+    ; nightly     - Tests that should only run in nightly builds (uses large models or long running)
     ; samples     - Tests related to the sample models.
     ; llm         - Tests related to large language models.
     ; whisper     - Tests related to the Whisper model.
@@ -12,8 +12,8 @@ markers =
     ; vlm         - Tests related to the VLM model.
     ; rag         - Tests related to the RAG components.
     ; speech_generation - Tests related to text-to-speech generation
-    precommit
     real_models
+    nightly
     samples
     llm
     whisper
@@ -24,4 +24,4 @@ markers =
     rag
     speech_generation
 
-addopts = -m precommit
+addopts = -m "not real_models and not nightly"

tests/python_tests/pytest.ini

@@ -32,7 +32,6 @@ def teardown_class(self):
 
     @pytest.mark.speech_generation
     @pytest.mark.samples
-    @pytest.mark.precommit
     @pytest.mark.parametrize("convert_model", ["tiny-random-SpeechT5ForTextToSpeech"], indirect=True)
     @pytest.mark.parametrize("input_prompt", ["Hello everyone"])
     def test_sample_text_to_speech(self, convert_model, input_prompt):
@@ -54,7 +53,6 @@ def test_sample_text_to_speech(self, convert_model, input_prompt):
 
     @pytest.mark.speech_generation
     @pytest.mark.samples
-    @pytest.mark.precommit
     @pytest.mark.parametrize("convert_model", ["tiny-random-SpeechT5ForTextToSpeech"], indirect=True)
     @pytest.mark.parametrize("input_prompt", ["Test text to speech without speaker embedding file"])
     def test_sample_text_to_speech_no_speaker_embedding_file(self, convert_model, input_prompt):

tests/python_tests/samples/test_text2speech.py

@@ -36,9 +36,8 @@ def read_models_list(file_name: str):
             models.append(model_name)
     return models
 
-@pytest.mark.precommit
-@pytest.mark.parametrize("model_id", read_models_list(os.path.join(os.path.dirname(os.path.realpath(__file__)), "models", "precommit")))
-def test_e2e_precommit(model_id):
+@pytest.mark.parametrize("model_id", read_models_list(os.path.join(os.path.dirname(os.path.realpath(__file__)), "models", "lightweight")))
+def test_e2e_lightweight_models(model_id):
     prompts, generation_configs = get_test_dataset()
     generate_and_compare(prompts=prompts,
                          generation_config=generation_configs,
@@ -73,7 +72,6 @@ def test_e2e_real_models(model_id):
 ]
 @pytest.mark.parametrize("generation_config", test_configs)
 @pytest.mark.parametrize("prompt", batched_prompts[1:])  # num_beams=15 diverges on the first prompt.
-@pytest.mark.precommit
 @pytest.mark.skip(reason="CVS-162891: Fix test_continuous_batching_vs_stateful tests after we started to compare cb vs sdpa")
 def test_continuous_batching_vs_stateful(prompt, generation_config):
     model_id = "facebook/opt-125m"
@@ -93,7 +91,6 @@ def test_continuous_batching_vs_stateful(prompt, generation_config):
 
 prompts = ['The Sun is yellow because', 'Difference between Jupiter and Mars is that', 'table is made of']
 @pytest.mark.parametrize("prompt", prompts)
-@pytest.mark.precommit
 def test_cb_streamer_vs_return_vs_stateful(prompt):
     model_id = "facebook/opt-125m"
     _, _, models_path = download_and_convert_model(model_id)
@@ -124,7 +121,6 @@ def test_cb_streamer_vs_return_vs_stateful(prompt):
 @pytest.mark.parametrize("input_type", [
     GenerationChatInputsType.STRING,
     GenerationChatInputsType.CHAT_HISTORY])
-@pytest.mark.precommit
 def test_chat_scenario_vs_stateful(model_id, generation_config_kwargs: dict, pipeline_type, input_type: GenerationChatInputsType):
     _, _, models_path = download_and_convert_model(model_id)
 
@@ -175,7 +171,6 @@ def test_chat_scenario_vs_stateful(model_id, generation_config_kwargs: dict, pip
 @pytest.mark.parametrize("generation_config_kwargs", generation_configs)
 @pytest.mark.parametrize("model_id", get_chat_models_list())
 @pytest.mark.parametrize("pipeline_type", [PipelineType.CONTINUOUS_BATCHING, PipelineType.SPECULATIVE_DECODING, PipelineType.PROMPT_LOOKUP_DECODING,])
-@pytest.mark.precommit
 def test_continuous_batching_add_request_health_check(model_id, generation_config_kwargs: dict, pipeline_type):
     _, _, models_path = download_and_convert_model(model_id)
 
@@ -206,7 +201,6 @@ def test_continuous_batching_add_request_health_check(model_id, generation_confi
 @pytest.mark.parametrize("generation_config_kwargs", invalid_generation_configs)
 @pytest.mark.parametrize("model_id", get_chat_models_list())
 @pytest.mark.parametrize("pipeline_type", [PipelineType.CONTINUOUS_BATCHING, PipelineType.SPECULATIVE_DECODING, PipelineType.PROMPT_LOOKUP_DECODING,])
-@pytest.mark.precommit
 def test_continuous_batching_add_request_fails(model_id, generation_config_kwargs: dict, pipeline_type):
     _, _, models_path = download_and_convert_model(model_id)
 
@@ -228,7 +222,6 @@ def test_continuous_batching_add_request_fails(model_id, generation_config_kwarg
 #
 
 # todo: iefode: bug reproducer!!!
-@pytest.mark.precommit
 @pytest.mark.parametrize("sampling_config", [get_greedy(), get_beam_search(), get_multinomial_all_parameters()],
                          ids=["greedy", "beam_search", "multinomial_all_parameters"])
 def test_post_oom_health(sampling_config):
@@ -289,7 +282,6 @@ def get_beam_search_seq_len_300() -> GenerationConfig:
                          ({"num_kv_blocks": 100, "dynamic_split_fuse": True}, get_beam_search_seq_len_300()),
                          ({"num_kv_blocks": 100, "dynamic_split_fuse": False}, get_beam_search_seq_len_300())]
 @pytest.mark.parametrize("params", scheduler_params_list)
-@pytest.mark.precommit
 def test_preemption(params):
     model_id = "facebook/opt-125m"
     scheduler_params = params[0]
@@ -342,7 +334,6 @@ def test_preemption(params):
 
 # todo: Anastasiia Pnevskaya: fix the test because it is hanging according max_new_tokens = std::numeric_limits<std::size_t>::max()
 @pytest.mark.parametrize("dynamic_split_fuse", [True, False])
-@pytest.mark.precommit
 @pytest.mark.skip(reason="Random sampling results are non deterministic due to: discrete_distribution impl depends on platform, model inference results may depend on CPU. Test passes on CI but fails locally.")
 def test_preemption_with_multinomial(dynamic_split_fuse):
     generation_configs = multinomial_params.generation_config
@@ -425,7 +416,6 @@ def test_preemption_with_multinomial(dynamic_split_fuse):
 
 
 @pytest.mark.parametrize("dynamic_split_fuse", [True, False])
-@pytest.mark.precommit
 @pytest.mark.skip(reason="Random sampling results are non deterministic due to: discrete_distribution impl depends on platform, model inference results may depend on CPU. Test passes on CI but fails locally.")
 def test_preemption_with_multinomial_n_seq(dynamic_split_fuse):
     model_id : str = "facebook/opt-125m"
@@ -442,7 +432,6 @@ def test_preemption_with_multinomial_n_seq(dynamic_split_fuse):
 
 
 @pytest.mark.parametrize("pipeline_type", [PipelineType.PROMPT_LOOKUP_DECODING])
-@pytest.mark.precommit
 def test_dynamic_split_fuse_doesnt_affect_generated_text(pipeline_type):
     model_id : str = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
     _, _, models_path = download_and_convert_model(model_id)
@@ -496,7 +485,6 @@ def run_extended_perf_metrics_collection(model_id, generation_config: Generation
 
 
 @pytest.mark.parametrize("pipeline_type", [PipelineType.PAGED_ATTENTION, PipelineType.SPECULATIVE_DECODING])
-@pytest.mark.precommit
 def test_speculative_decoding_extended_perf_metrics(pipeline_type):
     import time
     start_time = time.perf_counter()