[GENAI]Introduce add_extension to genai.

src/cpp/src/continuous_batching/pipeline.cpp

@@ -46,6 +46,7 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline( const std::filesystem::p
     auto draft_model_desr = utils::extract_draft_model_from_config(properties_without_draft_model);
     auto is_prompt_lookup_enabled = extract_prompt_lookup_from_config(properties_without_draft_model);
 
+    utils::add_extensions_to_core(properties_without_draft_model);
     auto model = utils::read_model(models_path, properties);
     auto [properties_without_draft_model_without_gguf, enable_save_ov_model] = utils::extract_gguf_properties(properties_without_draft_model);
     properties_without_draft_model_without_gguf[ov::cache_model_path.name()] = models_path;
@@ -88,6 +89,7 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(
     auto draft_model_desr = utils::extract_draft_model_from_config(properties_without_draft_model);
     auto is_prompt_lookup_enabled = extract_prompt_lookup_from_config(properties_without_draft_model);
 
+    utils::add_extensions_to_core(properties_without_draft_model);
     auto model = utils::read_model(models_path, properties_without_draft_model);
     auto [properties_without_draft_model_without_gguf, enable_save_ov_model] = utils::extract_gguf_properties(properties_without_draft_model);
     properties_without_draft_model_without_gguf[ov::cache_model_path.name()] = models_path;
@@ -131,6 +133,8 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(
     auto properties_without_draft_model = properties;
     auto draft_model_desr = utils::extract_draft_model_from_config(properties_without_draft_model);
     auto is_prompt_lookup_enabled = extract_prompt_lookup_from_config(properties_without_draft_model);
+
+    utils::add_extensions_to_core(properties_without_draft_model);
     auto model = utils::singleton_core().read_model(model_str, weights_tensor);
 
     auto rt_info = model->get_rt_info();
@@ -177,6 +181,8 @@ ContinuousBatchingPipeline::ContinuousBatchingPipeline(
     auto draft_model_desr = utils::extract_draft_model_from_config(properties_without_draft_model);
     auto is_prompt_lookup_enabled = extract_prompt_lookup_from_config(properties_without_draft_model);
     auto model_pair = utils::get_model_weights_pair(models_map, "language");
+
+    utils::add_extensions_to_core(properties_without_draft_model);
     auto model = utils::singleton_core().read_model(model_pair.first, model_pair.second);
 
     auto rt_info = model->get_rt_info();

src/cpp/src/llm/pipeline.cpp

@@ -259,6 +259,7 @@ ov::genai::LLMPipeline::LLMPipeline(
 
     bool is_npu_requested = ov::genai::utils::is_npu_requested(device, user_properties);
     auto [properties, attention_backend] = utils::extract_attention_backend(user_properties, is_npu_requested);
+    utils::add_extensions_to_core(properties);
 
     if (is_npu_requested) {
         m_pimpl = StatefulPipeline::create(

src/cpp/src/llm/pipeline_stateful.cpp

@@ -78,7 +78,9 @@ StatefulLLMPipeline::StatefulLLMPipeline(
         m_max_prompt_len = kv_desc.max_prompt_len;
         m_max_kv_cache_size = kv_desc.max_prompt_len + kv_desc.min_response_len;
     } else {
-       compiled_model = utils::singleton_core().compile_model(model, device, *filtered_properties);
+        auto properties_without_extensions = *filtered_properties;
+        utils::add_extensions_to_core(properties_without_extensions);
+        compiled_model = utils::singleton_core().compile_model(model, device, properties_without_extensions);
     }
     m_model_runner = compiled_model.create_infer_request();
     ov::genai::utils::print_compiled_model_properties(compiled_model, "Stateful LLM model");

src/cpp/src/utils.cpp

@@ -718,6 +718,17 @@ std::pair<ov::AnyMap, std::string> extract_attention_backend(const ov::AnyMap& e
     return {properties, attention_backend};
 };
 
+void add_extensions_to_core(ov::AnyMap& properties) {
+    auto it = properties.find(EXTENSIONS_ARG_NAME);
+    if (it != properties.end()) {
+        auto extensions = it->second.as<std::vector<std::string>>();
+        for (const auto& extension : extensions) {
+            singleton_core().add_extension(extension);
+        }
+        properties.erase(it);
+    }
+}
+
 void release_core_plugin(const std::string& device) {
     try {
         singleton_core().unload_plugin(device);

src/cpp/src/utils.hpp

@@ -103,6 +103,7 @@ void read_anymap_param(const ov::AnyMap& config_map, const std::string& name, T&
 const std::string STREAMER_ARG_NAME = "streamer";
 const std::string CONFIG_ARG_NAME = "generation_config";
 const std::string DRAFT_MODEL_ARG_NAME = "draft_model";
+const std::string EXTENSIONS_ARG_NAME = "EXTENSIONS";
 
 template<typename Config = ov::genai::GenerationConfig>
 Config from_config_json_if_exists(const std::filesystem::path& models_path, const char config_name[] = "generation_config.json") {
@@ -286,6 +287,8 @@ bool explicitly_requires_paged_attention(const ov::AnyMap& properties, bool is_n
 
 std::pair<ov::AnyMap, std::string> extract_attention_backend(const ov::AnyMap& external_properties, bool is_npu_requested = false);
 
+void add_extensions_to_core(ov::AnyMap& properties);
+
 void save_openvino_model(const std::shared_ptr<ov::Model>& model, const std::string& save_path, bool compress_to_fp16);
 
 ov::Tensor merge_text_and_image_embeddings_llava(const ov::Tensor& input_ids, ov::Tensor& text_embeds, const std::vector<ov::Tensor>& image_embeds, int64_t image_token_id);

src/cpp/src/visual_language/embedding_model.cpp

@@ -45,11 +45,13 @@ EmbeddingsModel::EmbeddingsModel(const std::filesystem::path& model_dir,
                                  const std::string& device,
                                  const ov::AnyMap& properties) {
     ov::Core core = utils::singleton_core();
-    std::shared_ptr<ov::Model> m_model = core.read_model(model_dir / "openvino_text_embeddings_model.xml", {}, properties);
+    auto properties_copy = properties;
+    utils::add_extensions_to_core(properties_copy);
+    std::shared_ptr<ov::Model> m_model = core.read_model(model_dir / "openvino_text_embeddings_model.xml", {}, properties_copy);
     // apply embedding postprocessing step by merging them into the model
     merge_postprocess(m_model, scale_emb);
 
-    ov::CompiledModel compiled_model = core.compile_model(m_model, device, properties);
+    ov::CompiledModel compiled_model = core.compile_model(m_model, device, properties_copy);
     ov::genai::utils::print_compiled_model_properties(compiled_model, "text embeddings model");
     m_embeddings_requests_queue = init(compiled_model);
 }
@@ -60,11 +62,13 @@ EmbeddingsModel::EmbeddingsModel(const std::string& model,
                                  const std::string& device,
                                  const ov::AnyMap& properties) {
     ov::Core core = utils::singleton_core();
+    auto properties_copy = properties;
+    utils::add_extensions_to_core(properties_copy);
     std::shared_ptr<ov::Model> m_model = core.read_model(model, weights);
     // apply embedding postprocessing step by merging them into the model
     merge_postprocess(m_model, scale_emb);
 
-    ov::CompiledModel compiled_model = core.compile_model(m_model, device, properties);
+    ov::CompiledModel compiled_model = core.compile_model(m_model, device, properties_copy);
     m_embeddings_requests_queue = init(compiled_model);
 }
 

src/cpp/src/visual_language/pipeline.cpp

@@ -82,7 +82,8 @@ class VLMPipeline::VLMPipelineImpl : public VLMPipelineBase{
 
         auto properties_copy = properties;
         auto language_model_path = models_dir / "openvino_language_model.xml";
-        auto language_model =  utils::singleton_core().read_model(language_model_path, {}, properties_copy);
+        utils::add_extensions_to_core(properties_copy);
+        auto language_model = utils::singleton_core().read_model(language_model_path, {}, properties_copy);
         auto kv_pos = ov::genai::utils::get_kv_axes_pos(language_model);
 
         // In case user provided properties per-device
@@ -157,8 +158,11 @@ class VLMPipeline::VLMPipelineImpl : public VLMPipelineBase{
         m_embedding = m_inputs_embedder->get_embedding_model();
 
         auto m_language_pair = utils::get_model_weights_pair(models_map, "language");
+        auto properties_without_extensions = properties;
+        utils::add_extensions_to_core(properties_without_extensions);
+
         m_language = utils::singleton_core().compile_model(
-            m_language_pair.first, m_language_pair.second, device, properties
+            m_language_pair.first, m_language_pair.second, device, properties_without_extensions
         ).create_infer_request();
 
         m_language.get_tensor("attention_mask").set_shape({1, 0});

src/cpp/src/visual_language/qwen2vl/classes.cpp

@@ -660,8 +660,10 @@ VisionEncoderQwen2VL::VisionEncoderQwen2VL(const std::filesystem::path& model_di
     : VisionEncoder(model_dir, device, properties),
       use_ov_image_preprocess(check_image_preprocess_env()) {
     if (use_ov_image_preprocess) {
+        auto properties_without_extensions = properties;
+        utils::add_extensions_to_core(properties_without_extensions);
         auto model_org = utils::singleton_core().read_model(model_dir / "openvino_vision_embeddings_model.xml");
-        m_ireq_queue_vision_encoder = create_vision_encoder_ireq(model_org, m_processor_config, device, properties);
+        m_ireq_queue_vision_encoder = create_vision_encoder_ireq(model_org, m_processor_config, device, properties_without_extensions);
     }
 }
 
@@ -674,8 +676,10 @@ VisionEncoderQwen2VL::VisionEncoderQwen2VL(const ModelsMap& models_map,
     if (use_ov_image_preprocess) {
         const auto& [vision_encoder_model, vision_encoder_weights] =
             utils::get_model_weights_pair(models_map, "vision_embeddings");
+        auto properties_without_extensions = properties;
+        utils::add_extensions_to_core(properties_without_extensions);
         auto model_org = utils::singleton_core().read_model(vision_encoder_model, vision_encoder_weights);
-        m_ireq_queue_vision_encoder = create_vision_encoder_ireq(model_org, m_processor_config, device, properties);
+        m_ireq_queue_vision_encoder = create_vision_encoder_ireq(model_org, m_processor_config, device, properties_without_extensions);
     }
 }
 
@@ -923,10 +927,12 @@ InputsEmbedderQwen2VL::InputsEmbedderQwen2VL(
     const std::string& device,
     const ov::AnyMap device_config) :
     IInputsEmbedder(vlm_config, model_dir, device, device_config) {
+    auto properties_without_extensions = device_config;
+    utils::add_extensions_to_core(properties_without_extensions);
     auto model = utils::singleton_core().read_model(model_dir / "openvino_vision_embeddings_merger_model.xml");
     utils::request_vl_sdpa_transformations(model);
 
-    auto compiled_model = utils::singleton_core().compile_model(model, device, device_config);
+    auto compiled_model = utils::singleton_core().compile_model(model, device, properties_without_extensions);
 
     m_with_cu_seqlens_input = utils::check_vl_sdpa_transformations(compiled_model);
     ov::genai::utils::print_compiled_model_properties(compiled_model,
@@ -952,14 +958,16 @@ InputsEmbedderQwen2VL::InputsEmbedderQwen2VL(
     const std::string& device,
     const ov::AnyMap device_config) :
     IInputsEmbedder(vlm_config, models_map, tokenizer, config_dir_path, device, device_config) {
+    auto properties_without_extensions = device_config;
+    utils::add_extensions_to_core(properties_without_extensions);
     auto model = utils::singleton_core().read_model(
         utils::get_model_weights_pair(models_map, "vision_embeddings_merger").first,
         utils::get_model_weights_pair(models_map, "vision_embeddings_merger").second);
     utils::request_vl_sdpa_transformations(model);
 
     auto compiled_model = utils::singleton_core().compile_model(model,
         device,
-        device_config
+        properties_without_extensions
     );
 
     m_with_cu_seqlens_input = utils::check_vl_sdpa_transformations(compiled_model);

src/python/openvino_genai/py_openvino_genai.pyi

@@ -4465,4 +4465,4 @@ def draft_model(models_path: os.PathLike | str | bytes, device: str = '', **kwar
 def get_version() -> str:
     """
     OpenVINO GenAI version
-    """
+    """

tests/python_tests/test_continuous_batching.py

@@ -530,3 +530,17 @@ def test_speculative_decoding_extended_perf_metrics(pipeline_type):
             assert std_gen_duration == 0
     else:
         assert extended_perf_metrics is None
+
+
+@pytest.mark.precommit
+def test_continuous_batching_add_extension():
+    model_id = 'katuni4ka/tiny-random-phi3'
+    _, _, models_path = download_and_convert_model(model_id)
+
+    scheduler_config = SchedulerConfig()
+
+    properties = {"EXTENSIONS": ["fake_path"]}
+
+    with pytest.raises(RuntimeError) as exc_info:
+        ContinuousBatchingPipeline(models_path, scheduler_config, "CPU", properties)
+    assert "Cannot find entry point to the extension library" in str(exc_info.value)

tests/python_tests/test_llm_pipeline.py

@@ -809,3 +809,14 @@ def py_streamer(py_str: str):
     else:
         assert it_cnt > 0
 
+
+@pytest.mark.precommit
+def test_llm_pipeline_add_extension():
+    model_id = "katuni4ka/tiny-random-phi3"
+    _, _, models_path = download_and_convert_model(model_id)
+
+    properties = {"EXTENSIONS": ["fake_path"]}
+
+    with pytest.raises(RuntimeError) as exc_info:
+        ov_genai.LLMPipeline(models_path, "CPU", **properties)
+    assert "Cannot find entry point to the extension library" in str(exc_info.value)

tests/python_tests/test_vlm_pipeline.py

@@ -304,12 +304,12 @@ def ov_pipe_model(request: pytest.FixtureRequest) -> VlmModelInfo:
     ids=lambda p: f"{p[0]}/{p[1]}",
     indirect=["ov_pipe_model"],
 )
-    
+
 @pytest.fixture(scope="module")
 def ov_continious_batching_pipe() -> ContinuousBatchingPipeline:
     models_path = _get_ov_model(MODEL_IDS[0])
     return ContinuousBatchingPipeline(models_path, SchedulerConfig(), "CPU")
-    
+
 @pytest.fixture(scope="module")
 def ov_continious_batching_pipe_gemma() -> ContinuousBatchingPipeline:
     models_path = _get_ov_model(MODEL_IDS[8])
@@ -1366,7 +1366,7 @@ def test_model_tags_missing_native(ov_pipe_model: VlmModelInfo):
 
     with pytest.raises(RuntimeError):
         ov_pipe.generate(image_tag(0))
-            
+
 
 @pytest.mark.parametrize(
     "ov_pipe_model,has_image,has_video",
@@ -1508,3 +1508,14 @@ def get_nanollava_processor():
     genai_text = genai_output.texts[0]
 
     assert optimum_text == genai_text
+
+@pytest.mark.precommit
+def test_vlm_pipeline_add_extension():
+    model_id = VIDEO_MODEL_IDS[1]
+    models_path = _get_ov_model(model_id)
+
+    properties = {"EXTENSIONS": ["fake_path"]}
+
+    with pytest.raises(RuntimeError) as exc_info:
+        VLMPipeline(models_path, "CPU", config=properties)
+    assert "Cannot find entry point to the extension library" in str(exc_info.value)