[OVEP] OpenVINO EP Features for ORT 1.23 Release

onnxruntime/core/providers/openvino/backend_manager.cc

@@ -183,9 +183,13 @@ BackendManager::BackendManager(SessionContext& session_context,
   }
   if (session_context_.so_context_enable &&
       (subgraph_context_.is_ep_ctx_ovir_encapsulated || !subgraph_context_.is_ep_ctx_graph)) {
-    auto status = onnxruntime::openvino_ep::BackendManager::ExportCompiledBlobAsEPCtxNode(subgraph);
-    if (!status.IsOK()) {
-      ORT_THROW(status);
+    if (concrete_backend_) {
+      auto status = onnxruntime::openvino_ep::BackendManager::ExportCompiledBlobAsEPCtxNode(subgraph);
+      if (!status.IsOK()) {
+        ORT_THROW(status);
+      }
+    } else {
+      ORT_THROW("[OpenVINO-EP] Cannot export compiled blob as EPCtx Node: Backend not initialized.");
     }
   }
 }
@@ -660,6 +664,7 @@ void BackendManager::Compute(OrtKernelContext* context) {
 }
 
 void BackendManager::ShutdownBackendManager() {
+  std::unique_lock<std::mutex> lock(mutex_);
   backend_map_.clear();
   concrete_backend_.reset();
 }

onnxruntime/core/providers/openvino/backend_utils.h

@@ -37,7 +37,7 @@
     std::transform(ort_shape.begin(), ort_shape.end(), ov_shape.begin(), [](int64_t dim) {
       return dim == -1 ? ov::Dimension::dynamic() : ov::Dimension(dim);
     });
-    return ov::PartialShape(ov_shape);
+    return ov::PartialShape(std::move(ov_shape));
   }
 
   static ort_shape_t ToOrtShape(const ov::PartialShape& ov_shape) {

onnxruntime/core/providers/openvino/backends/basic_backend.cc

@@ -44,7 +44,7 @@ BasicBackend::BasicBackend(std::unique_ptr<ONNX_NAMESPACE::ModelProto>& model_pr
         // model_file_path will use so_context_file_path if the onnx_model_path_name is not available,
         // especially in case of CreateSessionFormArray() where user must explicitly
         // specify absolute path for so_context_file_path.
-        auto model_file_path = [this]() {
+        auto model_file_path = [this]() -> const std::filesystem::path& {
           if (!session_context_.onnx_model_path_name.empty() &&
               std::filesystem::exists(session_context_.onnx_model_path_name)) return session_context_.onnx_model_path_name;
 

onnxruntime/core/providers/openvino/exported_symbols.lst

@@ -1 +1,3 @@
 _GetProvider
+_CreateEpFactories
+_ReleaseEpFactory

onnxruntime/core/providers/openvino/openvino_execution_provider.cc

@@ -118,7 +118,7 @@ common::Status OpenVINOExecutionProvider::Compile(
     fs::path metadata_file_path = context_model_file_path.parent_path() / metadata_filename;
     std::ifstream file(metadata_file_path, std::ios::binary);
     ORT_RETURN_IF_NOT(file, "Metadata file was not found: " + metadata_file_path.string());
-    shared_context_->shared_weights.metadata_filepath = metadata_file_path;
+    shared_context_->shared_weights.metadata_filepath = std::move(metadata_file_path);
     file >> metadata;
   }
 

onnxruntime/core/providers/openvino/openvino_parser_utils.cc

@@ -142,7 +142,7 @@
   }
 
   // Process each tensor definition e.g. "input_1[1..5, 2, 3..4],data[1,2,3]"
-  for (std::sregex_iterator i = tensor_begin; i != tensor_end; ++i) {
+  for (std::sregex_iterator i = std::move(tensor_begin); i != tensor_end; ++i) {
     std::smatch tensor_match = *i;
 
     // Extract tensor name and trim whitespace
@@ -165,7 +165,7 @@
     auto dim_end = std::sregex_iterator();
 
     // Process each dimension
-    for (std::sregex_iterator j = dim_begin; j != dim_end; ++j) {
+    for (std::sregex_iterator j = std::move(dim_begin); j != dim_end; ++j) {
       std::smatch dim_match = *j;
       std::string dim_value = dim_match[1].str();
 
@@ -190,7 +190,7 @@
     }
 
     // Store parsed shape in result map
-    parsed_shape_map[tensor_name] = ov::PartialShape(dimensions);
+    parsed_shape_map[tensor_name] = ov::PartialShape(std::move(dimensions));
   }
 
   return parsed_shape_map;

onnxruntime/core/providers/openvino/openvino_provider_factory.cc

@@ -403,6 +403,17 @@
     return ov_ep;
   }
 
+  // This is called during session creation when AppendExecutionProvider_V2 is used.
+  // This one is called because ParseProviderInfo / ParseConfigOptions, etc. are already
+  // performed in CreateIExecutionProvider, and so provider_info_ has already been populated.
+  std::unique_ptr<IExecutionProvider> CreateProvider_V2(const OrtSessionOptions& /*session_options*/,
+                                                        const OrtLogger& session_logger) {
+    ProviderInfo provider_info = provider_info_;
+    auto ov_ep = std::make_unique<OpenVINOExecutionProvider>(provider_info, shared_context_);
+    ov_ep->SetLogger(reinterpret_cast<const logging::Logger*>(&session_logger));
+    return ov_ep;
+  }
+
  private:
   ProviderInfo provider_info_;
   std::shared_ptr<SharedContext> shared_context_;
@@ -433,6 +444,115 @@
     return std::make_shared<OpenVINOProviderFactory>(pi, SharedContext::Get());
   }
 
+  Status CreateIExecutionProvider(const OrtHardwareDevice* const* /*devices*/,
+                                  const OrtKeyValuePairs* const* ep_metadata,
+                                  size_t num_devices,
+                                  ProviderOptions& provider_options,
+                                  const OrtSessionOptions& session_options,
+                                  const OrtLogger& logger,
+                                  std::unique_ptr<IExecutionProvider>& ep) override {
+    // Check if no devices are provided
+    if (num_devices == 0) {
+      return Status(common::ONNXRUNTIME, ORT_EP_FAIL, "No devices provided to CreateIExecutionProvider");
+    }
+
+    // For provider options that we don't support directly but are still supported through load_config,
+    // give some specific guidance & example about how to make use of the option through load_config.
+    const std::vector<std::pair<std::string, std::string>> block_and_advise_entries = {
+        {"cache_dir", "\"CACHE_DIR\": \"<filesystem_path>\""},
+        {"precision", "\"INFERENCE_PRECISION_HINT\": \"F32\""},
+        {"num_of_threads", "\"INFERENCE_NUM_THREADS\": \"1\""},
+        {"num_streams", "\"NUM_STREAMS\": \"1\""},
+        {"model_priority", "\"MODEL_PRIORITY\": \"LOW\""},
+        {"enable_opencl_throttling", "\"GPU\": {\"PLUGIN_THROTTLE\": \"1\"}"},
+        {"enable_qdq_optimizer", "\"NPU\": {\"NPU_QDQ_OPTIMIZATION\": \"YES\"}"}};
+
+    for (auto& block_and_advise_entry : block_and_advise_entries) {
+      if (provider_options.find(block_and_advise_entry.first) != provider_options.end()) {
+        std::string message = "OpenVINO EP: Option '" + block_and_advise_entry.first +
+                              "' cannot be set when using AppendExecutionProvider_V2. " +
+                              "It can instead be enabled by a load_config key / value pair. For example: " +
+                              block_and_advise_entry.second;
+        return Status(common::ONNXRUNTIME, ORT_INVALID_ARGUMENT, message);
+      }
+    }
+
+    // For the rest of the disallowed provider options, give a generic error message.
+    const std::vector<std::string> blocked_provider_keys = {
+        "device_type", "device_id", "device_luid", "context", "disable_dynamic_shapes"};
+
+    for (const auto& key : blocked_provider_keys) {
+      if (provider_options.find(key) != provider_options.end()) {
+        return Status(common::ONNXRUNTIME, ORT_INVALID_ARGUMENT,
+                      "OpenVINO EP: Option '" + key + "' cannot be set when using AppendExecutionProvider_V2.");
+      }
+    }
+
+    const char* ov_device_key = "ov_device";
+    const char* ov_meta_device_key = "ov_meta_device";
+
+    // Create a unique list of ov_devices that were passed in.
+    std::unordered_set<std::string_view> unique_ov_devices;
+    std::vector<std::string_view> ordered_unique_ov_devices;
+    for (size_t i = 0; i < num_devices; ++i) {
+      const auto& device_meta_data = ep_metadata[i];
+      auto ov_device_it = device_meta_data->Entries().find(ov_device_key);
+      if (ov_device_it == device_meta_data->Entries().end()) {
+        return Status(common::ONNXRUNTIME, ORT_INVALID_ARGUMENT, "OpenVINO EP device metadata not found.");
+      }
+      auto& ov_device = ov_device_it->second;
+
+      // Add to ordered_unique only if not already present
+      if (unique_ov_devices.insert(ov_device).second) {
+        ordered_unique_ov_devices.push_back(ov_device);
+      }
+    }
+
+    std::string ov_meta_device_type = "NONE";
+    {
+      auto ov_meta_device_it = ep_metadata[0]->Entries().find(ov_meta_device_key);
+      if (ov_meta_device_it != ep_metadata[0]->Entries().end()) {
+        ov_meta_device_type = ov_meta_device_it->second;
+      }
+    }
+
+    bool is_meta_device_factory = (ov_meta_device_type != "NONE");
+
+    if (ordered_unique_ov_devices.size() > 1 && !is_meta_device_factory) {
+      LOGS_DEFAULT(WARNING) << "[OpenVINO EP] Multiple devices were specified that are not OpenVINO meta devices. Using first ov_device only: " << ordered_unique_ov_devices.at(0);
+      ordered_unique_ov_devices.resize(1);  // Use only the first device if not a meta device factory
+    }
+
+    std::string ov_device_string;
+    if (is_meta_device_factory) {
+      // Build up a meta device string based on the devices that are passed in. E.g. AUTO:NPU,GPU.0,CPU
+      ov_device_string = ov_meta_device_type;
+      ov_device_string += ":";
+    }
+
+    bool prepend_comma = false;
+    for (const auto& ov_device : ordered_unique_ov_devices) {
+      if (prepend_comma) {
+        ov_device_string += ",";
+      }
+      ov_device_string += ov_device;
+      prepend_comma = true;
+    }
+
+    provider_options["device_type"] = ov_device_string;
+
+    // Parse provider info with the device type
+    ProviderInfo pi;
+    const auto& config_options = session_options.GetConfigOptions();
+    ParseProviderInfo(provider_options, &config_options, pi);
+    ParseConfigOptions(pi);
+
+    // Create and return the execution provider
+    auto factory = std::make_unique<OpenVINOProviderFactory>(pi, SharedContext::Get());
+    ep = factory->CreateProvider_V2(session_options, logger);
+    return Status::OK();
+  }
+
   void Initialize() override {
   }