From 3241f7cd4a54cae0c1d7b5345a94deed96dacf76 Mon Sep 17 00:00:00 2001 From: sfatima Date: Fri, 28 Feb 2025 16:38:37 +0530 Subject: [PATCH 1/2] Commit Changes --- .../core/providers/openvino/contexts.h | 1 + .../openvino/openvino_provider_factory.cc | 37 ++++++++++++------- .../core/session/provider_bridge_ort.cc | 12 ++++-- 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/onnxruntime/core/providers/openvino/contexts.h b/onnxruntime/core/providers/openvino/contexts.h index 216fc5b132696..a1a756a9baef7 100644 --- a/onnxruntime/core/providers/openvino/contexts.h +++ b/onnxruntime/core/providers/openvino/contexts.h @@ -101,6 +101,7 @@ struct ProviderInfo { bool so_context_embed_mode{false}; // ORT session option bool so_share_ep_contexts{false}; // ORT session option fs::path so_context_file_path{}; // ORT session option + const ConfigOptions* config_options{NULL}; }; // Holds context applicable to the entire EP instance. diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index 95e039f8b6d5f..a9b5be2e13deb 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -14,12 +14,14 @@ namespace onnxruntime { namespace openvino_ep { -void ParseConfigOptions(ProviderInfo& pi, const ConfigOptions& config_options) { - pi.so_disable_cpu_ep_fallback = config_options.GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; - pi.so_context_enable = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1"; - pi.so_context_embed_mode = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1"; - pi.so_share_ep_contexts = config_options.GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1"; - pi.so_context_file_path = config_options.GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); +void ParseConfigOptions(ProviderInfo& pi) { + if(pi.config_options==NULL) + return; + pi.so_disable_cpu_ep_fallback = pi.config_options->GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; + pi.so_context_enable = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1"; + pi.so_context_embed_mode = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1"; + pi.so_share_ep_contexts = pi.config_options->GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1"; + pi.so_context_file_path = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); } void* ParseUint64(const ProviderOptions& provider_options, std::string option_name) { @@ -166,6 +168,7 @@ struct OpenVINOProviderFactory : IExecutionProviderFactory { ~OpenVINOProviderFactory() override {} std::unique_ptr CreateProvider() override { + ParseConfigOptions(provider_info_); return std::make_unique(provider_info_, shared_context_); } @@ -184,13 +187,23 @@ struct OpenVINO_Provider : Provider { void* GetInfo() override { return &info_; } std::shared_ptr CreateExecutionProviderFactory(const void* void_params) override { - // Extract the void_params into ProviderOptions and ConfigOptions - using ConfigBuffer = std::pair; - const ConfigBuffer* buffer = reinterpret_cast(void_params); - const auto& provider_options = *buffer->first; - const auto& config_options = buffer->second; + if (void_params == nullptr) { + LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL options to CreateExecutionProviderFactory()"; + return nullptr; + } + + std::array pointers_array = *reinterpret_cast*>(void_params); + const ProviderOptions* provider_options_ptr = reinterpret_cast(pointers_array[0]); + const ConfigOptions* config_options = reinterpret_cast(pointers_array[1]); + + if(provider_options_ptr == NULL) { + LOGS_DEFAULT(ERROR) << "[OpenVINO EP] Passed NULL ProviderOptions to CreateExecutionProviderFactory()"; + return nullptr; + } + const ProviderOptions provider_options = *provider_options_ptr; ProviderInfo pi; + pi.config_options = config_options; std::string bool_flag = ""; @@ -326,8 +339,6 @@ struct OpenVINO_Provider : Provider { pi.disable_dynamic_shapes = ParseBooleanOption(provider_options, "disable_dynamic_shapes"); - ParseConfigOptions(pi, config_options); - // Always true for NPU plugin or when passed . if (pi.device_type.find("NPU") != std::string::npos) { pi.disable_dynamic_shapes = true; diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 1d25ceb9af8a3..77c6d4c371f69 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -1998,10 +1998,14 @@ std::shared_ptr QNNProviderFactoryCreator::Create(con std::shared_ptr OpenVINOProviderFactoryCreator::Create( const ProviderOptions* provider_options_map, const SessionOptions* session_options) { // Append session options applicable for EP to EP Provider options. - std::pair config_buffer = {provider_options_map, - session_options->config_options}; - const void* obj = reinterpret_cast(&config_buffer); - return s_library_openvino.Get().CreateExecutionProviderFactory(obj); + const ConfigOptions* config_options = nullptr; + if (session_options !=nullptr) { + config_options = &session_options->config_options; + } + + std::array configs_array = {provider_options_map, config_options}; + const void* arg = reinterpret_cast(&configs_array); + return s_library_openvino.Get().CreateExecutionProviderFactory(arg); } std::shared_ptr DnnlProviderFactoryCreator::Create(const OrtDnnlProviderOptions* dnnl_options) { From e4826ad45d52fa5f4da0c980c18af4d9d0e7fc15 Mon Sep 17 00:00:00 2001 From: sfatima Date: Fri, 28 Feb 2025 17:38:04 +0530 Subject: [PATCH 2/2] Changes to make sure to honor SessionOptions API Contract --- .../openvino/openvino_provider_factory.cc | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc index a9b5be2e13deb..c4fe16e035241 100644 --- a/onnxruntime/core/providers/openvino/openvino_provider_factory.cc +++ b/onnxruntime/core/providers/openvino/openvino_provider_factory.cc @@ -17,11 +17,19 @@ namespace openvino_ep { void ParseConfigOptions(ProviderInfo& pi) { if(pi.config_options==NULL) return; + pi.so_disable_cpu_ep_fallback = pi.config_options->GetConfigOrDefault(kOrtSessionOptionsDisableCPUEPFallback, "0") == "1"; pi.so_context_enable = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEnable, "0") == "1"; pi.so_context_embed_mode = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextEmbedMode, "0") == "1"; pi.so_share_ep_contexts = pi.config_options->GetConfigOrDefault(kOrtSessionOptionShareEpContexts, "0") == "1"; pi.so_context_file_path = pi.config_options->GetConfigOrDefault(kOrtSessionOptionEpContextFilePath, ""); + + if (pi.so_share_ep_contexts) { + ov::AnyMap map; + map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum"; + pi.load_config["NPU"] = std::move(map); + } + } void* ParseUint64(const ProviderOptions& provider_options, std::string option_name) { @@ -344,13 +352,6 @@ struct OpenVINO_Provider : Provider { pi.disable_dynamic_shapes = true; } - // Append values to config to support weight-as-inputs conversion for shared contexts - if (pi.so_share_ep_contexts) { - ov::AnyMap map; - map["NPU_COMPILATION_MODE_PARAMS"] = "enable-wd-blockarg-input=true compute-layers-with-higher-precision=Sqrt,Power,ReduceSum"; - pi.load_config["NPU"] = std::move(map); - } - return std::make_shared(pi, SharedContext::Get()); }