diff --git a/cmake/check_cuda.cmake b/cmake/check_cuda.cmake index 1eb20d5688..57dbce2de6 100644 --- a/cmake/check_cuda.cmake +++ b/cmake/check_cuda.cmake @@ -51,6 +51,11 @@ if((USE_CUDA OR USE_TRT_RTX) AND CMAKE_CUDA_COMPILER) "${GENERATORS_ROOT}/cuda/*.cuh" ) + # session_options.{h,cpp} are plain C++ (no CUDA kernels) and belong in the + # main onnxruntime-genai library (added via global_variables.cmake). Remove + # them from the CUDA library sources to avoid duplicate compilation. + list(FILTER generator_cudalib_srcs EXCLUDE REGEX ".*/cuda/session_options\\.(cpp|h)$") + add_compile_definitions(USE_CUDA=1) include_directories("${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}") elseif(USE_CUDA) diff --git a/cmake/global_variables.cmake b/cmake/global_variables.cmake index 5ab359139f..260c08be96 100644 --- a/cmake/global_variables.cmake +++ b/cmake/global_variables.cmake @@ -80,6 +80,16 @@ file(GLOB generator_srcs CONFIGURE_DEPENDS "${GENERATORS_ROOT}/openvino/*.cpp" "${GENERATORS_ROOT}/ryzenai/*.h" "${GENERATORS_ROOT}/ryzenai/*.cpp" + "${GENERATORS_ROOT}/cuda/session_options.h" + "${GENERATORS_ROOT}/cuda/session_options.cpp" + "${GENERATORS_ROOT}/nvtensorrtrtx/*.h" + "${GENERATORS_ROOT}/nvtensorrtrtx/*.cpp" + "${GENERATORS_ROOT}/vitisai/*.h" + "${GENERATORS_ROOT}/vitisai/*.cpp" + "${GENERATORS_ROOT}/rocm/session_options.h" + "${GENERATORS_ROOT}/rocm/session_options.cpp" + "${GENERATORS_ROOT}/dml/session_options.h" + "${GENERATORS_ROOT}/dml/session_options.cpp" "${MODELS_ROOT}/*.h" "${MODELS_ROOT}/*.cpp" "${ENGINE_ROOT}/*.h" diff --git a/cmake/ortlib.cmake b/cmake/ortlib.cmake index 0ec3e28cdd..a777da9e69 100644 --- a/cmake/ortlib.cmake +++ b/cmake/ortlib.cmake @@ -4,8 +4,6 @@ if(USE_WINML) message(STATUS "----- Building with WinML support ----- ") - add_compile_definitions(USE_WINML=1) - if(NOT DEFINED WINML_SDK_VERSION OR WINML_SDK_VERSION STREQUAL "") #set(WINML_SDK_VERSION "1.8.1065-experimental") # message(STATUS "WINML_SDK_VERSION not set, defaulting to ${WINML_SDK_VERSION}") @@ -17,7 +15,7 @@ if(USE_WINML) elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM STREQUAL "arm64X" OR CMAKE_GENERATOR_PLATFORM STREQUAL "arm64EC") set(ORT_PLATFORM "win-arm64") else() - message(FATACMAKE_GENERATOR_PLATFORML_ERROR "Unsupported platform for GenAI: ${CMAKE_GENERATOR_PLATFORM}") + message(FATAL_ERROR "Unsupported platform for GenAI: ${CMAKE_GENERATOR_PLATFORM}") return() endif() @@ -53,8 +51,6 @@ if(USE_WINML) file(COPY ${ORT_LIBS_1} DESTINATION "${ORT_HOME}/lib") message(STATUS "USE_WINML: ORT_HOME set to: ${ORT_HOME}") -else() - add_compile_definitions(USE_WINML=0) endif() if(ORT_HOME) diff --git a/src/cuda/session_options.cpp b/src/cuda/session_options.cpp new file mode 100644 index 0000000000..15d389934a --- /dev/null +++ b/src/cuda/session_options.cpp @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" +#include "../models/session_options.h" + +namespace Generators::CUDAExecutionProvider { + +namespace { + +void AppendProviderBridgeExecutionProvider( + OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + DeviceInterface*& device) { + auto ort_provider_options = OrtCUDAProviderOptionsV2::Create(); + std::vector keys, values; + + // Memory management settings + const char* arena_keys[] = { + "max_mem", + "arena_extend_strategy", + "initial_chunk_size_bytes", + "max_dead_bytes_per_chunk", + "initial_growth_chunk_size_bytes"}; + size_t arena_values[] = { + static_cast(0), + static_cast(-1), + static_cast(-1), + static_cast(-1), + static_cast(-1)}; + bool use_arena_management = false; + + for (auto& option : provider_options.options) { + auto it = std::find(std::begin(arena_keys), std::end(arena_keys), option.first); + + if (it == std::end(arena_keys)) { + keys.emplace_back(option.first.c_str()); + values.emplace_back(option.second.c_str()); + } else { + const size_t idx = std::distance(std::begin(arena_keys), it); + long long parsed_value = std::stoll(option.second); + if (parsed_value < -1) { + throw std::out_of_range("Arena configuration option value is out of range"); + } + arena_values[idx] = (parsed_value == -1) + ? static_cast(-1) + : static_cast(parsed_value); + use_arena_management = true; + } + } + ort_provider_options->Update(keys.data(), values.data(), keys.size()); + + // Device type determines the scoring device. + // Create and set our cudaStream_t + ort_provider_options->UpdateValue("user_compute_stream", device->GetCudaStream()); + + // Use fine-grained memory management of BFC Arena. + // The arena_cfg must outlive the AppendExecutionProvider_CUDA_V2 call below, + // so it is declared outside the if block. + std::unique_ptr arena_cfg; + if (use_arena_management) { + arena_cfg = OrtArenaCfg::Create(arena_keys, arena_values, std::size(arena_keys)); + ort_provider_options->UpdateValue("default_memory_arena_cfg", arena_cfg.get()); + } + + session_options.AppendExecutionProvider_CUDA_V2(*ort_provider_options); +} + +} // namespace + +void AddCudaStreamConfig(OrtSessionOptions& session_options, DeviceInterface* device, + const std::string& config_key) { + if (device) { + void* stream_ptr = device->GetCudaStream(); + std::stringstream stream_value; + stream_value << reinterpret_cast(stream_ptr); + session_options.AddConfigEntry(config_key.c_str(), stream_value.str().c_str()); + } +} + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& /*config*/, + bool /*disable_graph_capture*/) { + auto device = GetDeviceInterface(DeviceType::CUDA); + AddCudaStreamConfig(session_options, device); + // Try pre-registered plugin path first + if (!AppendExecutionProviderV2(session_options, provider_options, + DeviceType::CUDA, "CUDAExecutionProvider")) { + // Register the CUDA execution provider as a provider-bridge provider. + CUDAExecutionProvider::AppendProviderBridgeExecutionProvider( + session_options, provider_options, device); + } + + return device; +} + +} // namespace Generators::CUDAExecutionProvider diff --git a/src/cuda/session_options.h b/src/cuda/session_options.h new file mode 100644 index 0000000000..bf9c91cb0a --- /dev/null +++ b/src/cuda/session_options.h @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::CUDAExecutionProvider { + +// Writes the CUDA compute stream pointer (as a stringified integer) into a +// session config entry keyed by |config_key|. This is used by both the CUDA +// and NvTensorRtRtx providers so that the EP can share the same stream. +void AddCudaStreamConfig(OrtSessionOptions& session_options, DeviceInterface* device, + const std::string& config_key = "user_compute_stream"); + +// Registers the CUDA execution provider on |session_options|. Tries the V2 +// plugin path first; falls back to the provider-bridge (CUDA V2 options) path. +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::CUDAExecutionProvider diff --git a/src/dml/session_options.cpp b/src/dml/session_options.cpp new file mode 100644 index 0000000000..a7f8abcf64 --- /dev/null +++ b/src/dml/session_options.cpp @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" +#include "../models/session_options.h" + +#if USE_DML +#include "../dml/interface.h" +#endif +namespace Generators::DMLExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& /*config*/, + bool disable_graph_capture) { +#if USE_DML + if (!GetDmlInterface()) { + LUID device_luid{}; + LUID* p_device_luid{}; + uint32_t device_index{}; + uint32_t* p_device_index{}; + for (const auto& [name, value] : provider_options.options) { + if (name == "luid") { + if (auto separator_position = value.find(":"); separator_position != std::string::npos) { + device_luid.HighPart = std::stol(value.substr(0, separator_position)); + device_luid.LowPart = std::stol(value.substr(separator_position + 1)); + p_device_luid = &device_luid; + } + } else if (name == "device_index") { + device_index = std::stoi(value); + p_device_index = &device_index; + } + } + + InitDmlInterface(p_device_luid, p_device_index); + } + + // Non-decoder sessions (vision, speech, embedding) have control-flow nodes + // that are incompatible with graph capture, so the caller sets + // disable_graph_capture=true for those sessions. + if (!disable_graph_capture) { + session_options.AddConfigEntry("ep.dml.enable_graph_capture", "1"); + } + + SetDmlProvider(session_options); + + auto device = GetDeviceInterface(DeviceType::DML); // We use a DML allocator for input/output caches, but other tensors will use CPU tensors + return device; +#else + throw std::runtime_error("DML provider requested, but the installed GenAI has not been built with DML support"); +#endif +} + +} // namespace Generators::DMLExecutionProvider diff --git a/src/dml/session_options.h b/src/dml/session_options.h new file mode 100644 index 0000000000..4b73cd8dbc --- /dev/null +++ b/src/dml/session_options.h @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::DMLExecutionProvider { + +// Initialises the DML interface (if not already done), optionally enables graph +// capture, and registers the DirectML execution provider on |session_options|. +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::DMLExecutionProvider diff --git a/src/models/marian.cpp b/src/models/marian.cpp index 1da12617e9..5e2c0afa46 100644 --- a/src/models/marian.cpp +++ b/src/models/marian.cpp @@ -8,7 +8,7 @@ namespace Generators { MarianModel::MarianModel(std::unique_ptr config, OrtEnv& ort_env) : Model{std::move(config)} { encoder_session_options_ = OrtSessionOptions::Create(); - CreateSessionOptionsFromConfig(config_->model.encoder.session_options.has_value() ? config_->model.encoder.session_options.value() : config_->model.decoder.session_options, *encoder_session_options_, true, false); + CreateSessionOptionsFromConfig(config_->model.encoder.session_options.has_value() ? config_->model.encoder.session_options.value() : config_->model.decoder.session_options, *encoder_session_options_, true); session_encoder_ = CreateSession(ort_env, config_->model.encoder.filename, encoder_session_options_.get()); session_decoder_ = CreateSession(ort_env, config_->model.decoder.filename, session_options_.get()); diff --git a/src/models/model.cpp b/src/models/model.cpp index a0586ac8f7..c2790badc8 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -25,6 +25,7 @@ #include "../dml/interface.h" #include "../openvino/interface.h" #include "../ryzenai/interface.h" +#include "session_options.h" #if defined(_WIN32) #include @@ -392,489 +393,6 @@ int32_t Tokenizer::TokenToTokenId(const char* token) const { return token_id; } -/** - * @brief Creates profile shapes for NvTensorRtRtx execution provider optimization. - * - * This function generates profiles for TensorRT execution provider optimization. - * If multi-profile is enabled, it creates separate profiles for context and generation phases. - * If multi-profile is disabled, it creates a single profile with simple shapes. - * - */ -void ConfigureNvTensorRtRtxProfile(const Config& config, OrtSessionOptions& session_options, bool is_multi_profile_enabled) { - // Get model parameters from decoder config - const int num_layers = config.model.decoder.num_hidden_layers; - const int num_kv_heads = config.model.decoder.num_key_value_heads; - const int head_dim = config.model.decoder.head_size; - const int batch_size = config.search.batch_size * config.search.num_beams; - - // Get max context length from config - const int max_context_len = config.model.context_length; - - // Extract KV cache name patterns from decoder config - std::string_view past_key_pattern = config.model.decoder.inputs.past_key_names; - std::string_view past_value_pattern = config.model.decoder.inputs.past_value_names; - - // Helper function to add KV cache with sequence length - const auto add_key_value_cache_shapes = [](std::ostringstream& shapes, - int batch_size, - std::string_view key_pattern, - std::string_view value_pattern, - int seq_len, - int num_layers, - int num_kv_heads, - int head_dim) { - for (int i = 0; i < num_layers; i++) { - // Use the existing function to format the key/value names - const std::string key_name = ComposeKeyValueName(std::string(key_pattern), i); - const std::string value_name = ComposeKeyValueName(std::string(value_pattern), i); - - shapes << "," << key_name << ":" << batch_size << "x" << num_kv_heads << "x" << seq_len << "x" << head_dim; - shapes << "," << value_name << ":" << batch_size << "x" << num_kv_heads << "x" << seq_len << "x" << head_dim; - } - }; - - if (is_multi_profile_enabled) { - // Multi-profile mode: existing logic for context and generation phases - const int opt_context_len = config.model.context_length / 2; - const int min_seq_len = 1; - - // Helper function to add input shapes (input_ids, attention_mask, position_ids) - const auto add_input_shapes = [](std::ostringstream& shapes, int batch_size, int seq_len, bool append = false) { - if (append) shapes << ","; - shapes << Config::Defaults::InputIdsName << ":" << batch_size << "x" << seq_len << "," - << Config::Defaults::AttentionMaskName << ":" << batch_size << "x" << seq_len; - }; - - // Helper function to add generation phase input shapes - const auto add_generation_input_shapes = [](std::ostringstream& shapes, int batch_size, int context_len) { - shapes << "," << Config::Defaults::AttentionMaskName << ":" << batch_size << "x" << context_len << "," - << Config::Defaults::InputIdsName << ":" << batch_size << "x1"; - }; - - // Helper function to add empty KV cache shapes for all layers - const auto add_empty_key_value_cache_shapes = [](std::ostringstream& shapes, - int batch_size, - std::string_view key_pattern, - std::string_view value_pattern, - int num_layers, - int num_kv_heads, - int head_dim) { - for (int i = 0; i < num_layers; i++) { - // Use the existing function to format the key/value names - const std::string key_name = ComposeKeyValueName(std::string(key_pattern), i); - const std::string value_name = ComposeKeyValueName(std::string(value_pattern), i); - - shapes << "," << key_name << ":" << batch_size << "x" << num_kv_heads << "x0x" << head_dim; - shapes << "," << value_name << ":" << batch_size << "x" << num_kv_heads << "x0x" << head_dim; - } - }; - - std::ostringstream min_shapes, opt_shapes, max_shapes; - - // MIN SHAPES (context phase and first token generation) - add_input_shapes(min_shapes, batch_size, min_seq_len); - add_empty_key_value_cache_shapes(min_shapes, batch_size, past_key_pattern, past_value_pattern, num_layers, num_kv_heads, head_dim); - add_generation_input_shapes(min_shapes, batch_size, min_seq_len); - add_key_value_cache_shapes(min_shapes, batch_size, past_key_pattern, past_value_pattern, min_seq_len, num_layers, num_kv_heads, head_dim); - - // OPT SHAPES (prefill with medium context and generation after medium context) - add_input_shapes(opt_shapes, batch_size, opt_context_len); - add_empty_key_value_cache_shapes(opt_shapes, batch_size, past_key_pattern, past_value_pattern, num_layers, num_kv_heads, head_dim); - add_generation_input_shapes(opt_shapes, batch_size, opt_context_len); - add_key_value_cache_shapes(opt_shapes, batch_size, past_key_pattern, past_value_pattern, opt_context_len - 1, num_layers, num_kv_heads, head_dim); - - // MAX SHAPES (prefill with maximum context and generation after maximum context) - add_input_shapes(max_shapes, batch_size, max_context_len); - add_key_value_cache_shapes(max_shapes, batch_size, past_key_pattern, past_value_pattern, max_context_len - 1, num_layers, num_kv_heads, head_dim); - add_generation_input_shapes(max_shapes, batch_size, max_context_len); - add_key_value_cache_shapes(max_shapes, batch_size, past_key_pattern, past_value_pattern, max_context_len - 1, num_layers, num_kv_heads, head_dim); - - // Add the constructed profiles to session options - session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_min_shapes", min_shapes.str().c_str()); - session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_opt_shapes", opt_shapes.str().c_str()); - session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_max_shapes", max_shapes.str().c_str()); - } else { - // Single profile mode: simple shapes with batch_dim=[1,1,batch_size] and seq_dim=[1,1024,max_context_len] - std::ostringstream min_shapes, opt_shapes, max_shapes; - - // MIN SHAPES: batch_dim=1, seq_dim=1 - constexpr int min_context_len = 1; - constexpr int min_batch_size = 1; - min_shapes << Config::Defaults::InputIdsName << ":" << min_batch_size << "x" << min_context_len << "," - << Config::Defaults::AttentionMaskName << ":" << min_batch_size << "x" << min_context_len; - add_key_value_cache_shapes(min_shapes, min_batch_size, past_key_pattern, past_value_pattern, 0, num_layers, num_kv_heads, head_dim); - - // OPT SHAPES: batch_dim=1, seq_dim=1024 - const int opt_context_len = std::min(max_context_len / 2, 1024); // Use a reasonable opt context length - constexpr int opt_batch_size = 1; // Use a opt batch size of 1 - // keeping seq length to 1 as optimizing for the gen phase - opt_shapes << Config::Defaults::InputIdsName << ":" << opt_batch_size << "x" << 1 << "," - << Config::Defaults::AttentionMaskName << ":" << opt_batch_size << "x" << opt_context_len; - add_key_value_cache_shapes(opt_shapes, opt_batch_size, past_key_pattern, past_value_pattern, opt_context_len, num_layers, num_kv_heads, head_dim); - - // MAX SHAPES: seq_dim=max_context_len - max_shapes << Config::Defaults::InputIdsName << ":" << batch_size << "x" << max_context_len << "," - << Config::Defaults::AttentionMaskName << ":" << batch_size << "x" << max_context_len; - add_key_value_cache_shapes(max_shapes, batch_size, past_key_pattern, past_value_pattern, max_context_len, num_layers, num_kv_heads, head_dim); - - // Add the constructed profiles to session options - session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_min_shapes", min_shapes.str().c_str()); - session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_opt_shapes", opt_shapes.str().c_str()); - session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_max_shapes", max_shapes.str().c_str()); - } -} - -namespace { - -// Helper to check if a provider is pre-registered and get the matching EP device -const OrtEpDevice* FindPreRegisteredEpDevice(const std::string& ep_name) { - auto device_ptrs = GetOrtEnv().GetEpDevices(); - auto it = std::find_if(device_ptrs.begin(), device_ptrs.end(), - [&ep_name](const OrtEpDevice* device) { - return device->Name() == ep_name; - }); - return (it != device_ptrs.end()) ? *it : nullptr; -} - -// Helper to handle pre-registered plugin provider via V2 API -// Returns true if the provider was pre-registered and handled, false otherwise -bool IsProviderRegistered( - OrtSessionOptions& session_options, - const Config::ProviderOptions& provider_options, - DeviceType device_type, - const std::string& ep_name, - bool is_primary_session_options, - DeviceInterface*& p_device) { - const OrtEpDevice* ep_device = FindPreRegisteredEpDevice(ep_name); - if (!ep_device) return false; // Not pre-registered - - std::unordered_map options; - for (auto& option : provider_options.options) { - options.insert(option); - } - - if (is_primary_session_options) { - p_device = GetDeviceInterface(device_type); - if (p_device) { - void* stream_ptr = p_device->GetCudaStream(); - std::stringstream stream_value; - stream_value << reinterpret_cast(stream_ptr); - options.insert({"user_compute_stream", stream_value.str()}); - } - } - - std::vector ep_devices_ptrs = {ep_device}; - session_options.AppendExecutionProvider_V2(GetOrtEnv(), ep_devices_ptrs, options); - return true; // Handled -} - -} // namespace - -DeviceInterface* SetProviderSessionOptions(OrtSessionOptions& session_options, - const std::vector& providers, - const std::vector& provider_options_list, - bool is_primary_session_options, - bool disable_graph_capture, - const Config& config, - std::unique_ptr& arena_cfg) { - DeviceInterface* p_device{}; - - auto providers_list = providers; - if (!is_primary_session_options) { - // Providers specified in a non-primary provider options list are added - // to the primary providers. They are considered immutable and implicitly - // added as providers. - std::transform(provider_options_list.begin(), provider_options_list.end(), std::back_inserter(providers_list), - [](const auto& provider_options) { return provider_options.name; }); - } - - for (auto& provider : providers_list) { - auto provider_options_it = std::find_if(provider_options_list.begin(), provider_options_list.end(), - [&provider](const Config::ProviderOptions& po) { return po.name == provider; }); - - if (provider_options_it == provider_options_list.end()) { - throw std::runtime_error("Provider options not found for provider: " + provider); - } - const auto& provider_options = *provider_options_it; - - if (provider_options.name == "cuda") { - // Try pre-registered plugin path first - if (IsProviderRegistered(session_options, provider_options, - DeviceType::CUDA, "CUDAExecutionProvider", - is_primary_session_options, p_device)) { - continue; // Handled via V2 API, skip built-in path - } - - // Built-in CUDA path - { - auto ort_provider_options = OrtCUDAProviderOptionsV2::Create(); - std::vector keys, values; - - // Memory management settings - const char* arena_keys[] = {"max_mem", "arena_extend_strategy", "initial_chunk_size_bytes", "max_dead_bytes_per_chunk", "initial_growth_chunk_size_bytes"}; - size_t arena_values[] = {static_cast(0), static_cast(-1), static_cast(-1), static_cast(-1), static_cast(-1)}; - bool use_arena_management = false; - - for (auto& option : provider_options.options) { - auto it = std::find(std::begin(arena_keys), std::end(arena_keys), option.first); - - if (it == std::end(arena_keys)) { - keys.emplace_back(option.first.c_str()); - values.emplace_back(option.second.c_str()); - } else { - size_t idx = std::distance(std::begin(arena_keys), it); - arena_values[idx] = static_cast(std::stoull(option.second)); - use_arena_management = true; - } - } - ort_provider_options->Update(keys.data(), values.data(), keys.size()); - - // Device type determines the scoring device. - // Only use the primary session options to determine the device type - if (is_primary_session_options) { - p_device = GetDeviceInterface(DeviceType::CUDA); - - // Create and set our cudaStream_t - ort_provider_options->UpdateValue("user_compute_stream", p_device->GetCudaStream()); - } - - // Use fine-grained memory management of BFC Arena - if (use_arena_management) { - if (arena_cfg == nullptr) arena_cfg = OrtArenaCfg::Create(arena_keys, arena_values, 5); - ort_provider_options->UpdateValue("default_memory_arena_cfg", arena_cfg.get()); - } - - session_options.AppendExecutionProvider_CUDA_V2(*ort_provider_options); - } - } else if (provider_options.name == "rocm") { - OrtROCMProviderOptions ort_provider_options; - - std::vector keys, values; - for (auto& option : provider_options.options) { - keys.emplace_back(option.first.c_str()); - values.emplace_back(option.second.c_str()); - } - - Ort::ThrowOnError(Ort::api->UpdateROCMProviderOptions(&ort_provider_options, keys.data(), values.data(), keys.size())); - session_options.AppendExecutionProvider_ROCM(ort_provider_options); - } else if (provider_options.name == "DML") { -#if USE_DML - if (!GetDmlInterface()) { - LUID device_luid{}; - LUID* p_device_luid{}; - uint32_t device_index{}; - uint32_t* p_device_index{}; - for (const auto& [name, value] : provider_options.options) { - if (name == "luid") { - if (auto separator_position = value.find(":"); separator_position != std::string::npos) { - device_luid.HighPart = std::stol(value.substr(0, separator_position)); - device_luid.LowPart = std::stol(value.substr(separator_position + 1)); - p_device_luid = &device_luid; - } - } else if (name == "device_index") { - device_index = std::stoi(value); - p_device_index = &device_index; - } - } - - InitDmlInterface(p_device_luid, p_device_index); - } - - if (!disable_graph_capture) { - session_options.AddConfigEntry("ep.dml.enable_graph_capture", "1"); - } - - SetDmlProvider(session_options); - - if (is_primary_session_options) - p_device = GetDeviceInterface(DeviceType::DML); // We use a DML allocator for input/output caches, but other tensors will use CPU tensors -#else - throw std::runtime_error("DML provider requested, but the installed GenAI has not been built with DML support"); -#endif - } else if (provider_options.name == "OpenVINO") { - p_device = GetDeviceInterface(DeviceType::OpenVINO); - OpenVINO_AppendProviderOptions(session_options, config, provider_options); - } else if (provider_options.name == "RyzenAI") { - p_device = GetDeviceInterface(DeviceType::RyzenAI); - - session_options.AddConfigEntry("model_root", config.config_path.string().c_str()); - - GetRyzenAIInterface()->SetupProvider(session_options, provider_options.options); - } else if (provider_options.name == "NvTensorRtRtx") { - // Configure NvTensorRT-specific settings (needed for both pre-registered and built-in paths) - bool is_multi_profile_enabled = IsMultiProfileEnabled(config.model.decoder.session_options); - ConfigureNvTensorRtRtxProfile(config, session_options, is_multi_profile_enabled); - if (IsGraphCaptureEnabled(config.model.decoder.session_options)) { - session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.enable_cuda_graph", "1"); - } - - // Try pre-registered plugin path first - if (IsProviderRegistered(session_options, provider_options, - DeviceType::NvTensorRtRtx, "NvTensorRTRTXExecutionProvider", - is_primary_session_options, p_device)) { - continue; // Handled via V2 API - } - - // Built-in path: Configure stream via config entry (generic AppendExecutionProvider will be called below) - if (is_primary_session_options) { - p_device = GetDeviceInterface(DeviceType::NvTensorRtRtx); - if (p_device) { - void* stream_ptr = p_device->GetCudaStream(); - std::stringstream stream_value; - stream_value << reinterpret_cast(stream_ptr); - std::string stream_value_str = stream_value.str(); - session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.user_compute_stream", stream_value_str.c_str()); - } - } - // Fall through to generic provider registration below - } - - // Generic provider registration for all providers not handled by specific blocks above - // This handles: QNN, WebGPU, VitisAI, and NvTensorRtRtx (when not pre-registered) - // Note: cuda, rocm, DML, OpenVINO and RyzenAI are handled by their own specific blocks above - if (provider_options.name != "cuda" && provider_options.name != "rocm" && provider_options.name != "DML" && - provider_options.name != "OpenVINO" && provider_options.name != "RyzenAI") { - // Skip if NvTensorRtRtx was already handled via pre-registered plugin - if (provider_options.name == "NvTensorRtRtx" && FindPreRegisteredEpDevice("NvTensorRTRTXExecutionProvider")) { - continue; - } - - // For providers that go through the extensible AppendExecutionProvider API: - if (provider_options.name == "QNN") { - session_options.AddConfigEntry("ep.share_ep_contexts", "1"); - // TODO set device_type_ in a less hacky way. - // now, all QNN EP enable_htp_shared_memory_allocator option values had better be consistent... - // on the other hand, not sure if is_primary_session_options is the right thing to check here. - if (const auto opt_it = std::find_if(provider_options.options.begin(), provider_options.options.end(), - [](const auto& pair) { return pair.first == "enable_htp_shared_memory_allocator"; }); - opt_it != provider_options.options.end() && opt_it->second == "1") { - p_device = GetDeviceInterface(DeviceType::QNN); - } - } else if (provider_options.name == "WebGPU") - p_device = GetDeviceInterface(DeviceType::WEBGPU); - else if (provider_options.name == "VitisAI") { - session_options.AddConfigEntry("session.inter_op.allow_spinning", "0"); - session_options.AddConfigEntry("session.intra_op.allow_spinning", "0"); - session_options.AddConfigEntry("model_root", config.config_path.string().c_str()); - } - -#if USE_WINML - // Get device filtering config - Config::DeviceFilteringOptions resolved_device_filtering; - if (provider_options.device_filtering_options.has_value()) { - resolved_device_filtering = provider_options.device_filtering_options.value(); - } - - std::optional config_device_id = resolved_device_filtering.hardware_device_id; - std::optional config_vendor_id = resolved_device_filtering.hardware_vendor_id; - std::optional config_device_type_enum = resolved_device_filtering.hardware_device_type; - - // Match EP device with EP name in provider options and model device config - // include\onnxruntime\core\graph\constants.h - const static std::unordered_map s_providerNameToExecutionProvider{ - {"QNN", "QNNExecutionProvider"}, - {"WebGPU", "WebGpuExecutionProvider"}, - {"VitisAI", "VitisAIExecutionProvider"}, - {"NvTensorRtRtx", "NvTensorRTRTXExecutionProvider"}, - }; - std::string ep_name{}; - if (auto search = s_providerNameToExecutionProvider.find(provider_options.name); search != s_providerNameToExecutionProvider.end()) { - ep_name = search->second; - } - - size_t num_devices = 0; - const OrtEpDevice* const* device_ptrs = nullptr; - Ort::GetEpDevices(&GetOrtEnv(), &device_ptrs, &num_devices); - - std::vector ep_devices_ptrs; - ep_devices_ptrs.reserve(num_devices); - - for (size_t i = 0; i < num_devices; ++i) { - const OrtHardwareDevice* hardware_device = Ort::api->EpDevice_Device(device_ptrs[i]); - const uint32_t hardware_device_id = Ort::api->HardwareDevice_DeviceId(hardware_device); - const uint32_t hardware_vendor_id = Ort::api->HardwareDevice_VendorId(hardware_device); - const OrtHardwareDeviceType hardware_device_type = Ort::api->HardwareDevice_Type(hardware_device); - - bool hardware_device_id_matched = (!config_device_id.has_value()) || config_device_id.value() == hardware_device_id; - bool hardware_vendor_id_matched = (!config_vendor_id.has_value()) || config_vendor_id.value() == hardware_vendor_id; - bool hardware_device_type_matched = (!config_device_type_enum.has_value()) || - config_device_type_enum.value() == hardware_device_type; - - // Append matched EP device - if (Ort::api->EpDevice_EpName(device_ptrs[i]) == ep_name && - hardware_device_id_matched && - hardware_vendor_id_matched && - hardware_device_type_matched) { - ep_devices_ptrs.push_back(device_ptrs[i]); - // WinML Hotfix: DML and WebGPU EP factories currently only support one device at a time - if (provider_options.name == "DML" || provider_options.name == "WebGPU") { - break; - } - } - } - - // No need to append if we can't find a device. - if (!ep_devices_ptrs.empty()) { - std::vector keys, values; - for (auto& option : provider_options.options) { - // WinML Hotfix: remove backend_type and backend_path from QNN provider options - static const std::set qnn_options_to_remove{"backend_type", "backend_path"}; - if (provider_options.name == "QNN" && - qnn_options_to_remove.find(option.first) != qnn_options_to_remove.end()) { - continue; - } - - keys.emplace_back(option.first.c_str()); - values.emplace_back(option.second.c_str()); - } - - Ort::api->SessionOptionsAppendExecutionProvider_V2( - &session_options, - &GetOrtEnv(), - ep_devices_ptrs.data(), ep_devices_ptrs.size(), - keys.data(), values.data(), keys.size()); - } else if (provider_options.name == "NvTensorRtRtx") { - // Fallback to legacy API for built-in NvTensorRtRtx when no pre-registered device found - // This handles the case when using the built-in provider (not loaded as a plugin) - std::vector keys, values; - for (auto& option : provider_options.options) { - keys.emplace_back(option.first.c_str()); - values.emplace_back(option.second.c_str()); - } - session_options.AppendExecutionProvider(provider_options.name.c_str(), keys.data(), values.data(), keys.size()); - } -#else - std::vector keys, values; - - for (auto& option : provider_options.options) { - keys.emplace_back(option.first.c_str()); - values.emplace_back(option.second.c_str()); - } - session_options.AppendExecutionProvider(provider_options.name.c_str(), keys.data(), values.data(), keys.size()); -#endif -#if defined(_WIN32) - if (provider_options.name == "VitisAI") { - if (const auto opt_it = std::find_if(provider_options.options.begin(), provider_options.options.end(), - [](const auto& pair) { return pair.first == "external_ep_libray"; }); - opt_it != provider_options.options.end()) { - auto lib_name = opt_it->second; - auto lib = LoadLibrary(lib_name.c_str()); - if (const auto func = (void (*)(void*, const OrtApiBase*, void*, OrtEpFactory**, size_t, size_t*))GetProcAddress(lib, "CreateEpFactories")) { - OrtEpFactory* factory = nullptr; - size_t num = 1; - - func(nullptr, OrtGetApiBase(), nullptr, &factory, num, &num); - } - fs::path custom_ops_lib_path(lib_name); - session_options.RegisterCustomOpsLibrary(custom_ops_lib_path.c_str()); - } - } -#endif // WIN32 - } // end if (provider not cuda/rocm/DML) - } - return p_device; -} - // Trivial ONNX model that just returns a single float constant. Used below to create an OrtSession that // lets us get a device Ort::Allocator for each device type. This is necessary because the Ort::Allocator // needs to persist and is valid for the lifetime of this OrtSession. @@ -890,7 +408,7 @@ static const uint8_t g_trivial_model[] = { // the allocator used is not destroyed until last. This keeps the allocator around until exit, after all other memory // has been destroyed. Without this, we will crash in the OnnxRuntime BFCArena code when deleting tensors due to the // arena already being destroyed. -void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config, std::unique_ptr& arena_cfg) { +void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config) { // CPU Allocator is a special case, it's not in the owned 'allocator_device_' table below so we handle it separately // OpenVINO delegates to the CPU device allocator auto type = device.GetType(); @@ -920,7 +438,7 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config, std::uni provider_options_list.back().options.emplace_back("enable_htp_shared_memory_allocator", "1"); } const std::vector providers{device_type_names[static_cast(type)]}; - SetProviderSessionOptions(*session_options, providers, provider_options_list, true, false, config, arena_cfg); + SetProviderSessionOptions(*session_options, providers, provider_options_list, true, config); session_options->SetLogSeverityLevel(ORT_LOGGING_LEVEL_ERROR); // Errors only here, as warnings are not useful to the user allocator.session_ = OrtSession::Create(GetOrtEnv(), g_trivial_model, sizeof(g_trivial_model), session_options.get()); @@ -931,8 +449,28 @@ void EnsureDeviceOrtInit(DeviceInterface& device, const Config& config, std::uni // Get the allocator from the OrtSession for the DeviceType (it's called 'AllocatorCreate' but it's really 'AllocatorGet') auto name = device_memory_type_names[static_cast(type)]; - auto memory_info = OrtMemoryInfo::Create(name, OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemType::OrtMemTypeDefault); - allocator.allocator_ = Ort::Allocator::Create(*allocator.session_, *memory_info); + try { + auto memory_info = OrtMemoryInfo::Create(name, OrtAllocatorType::OrtDeviceAllocator, + 0, OrtMemType::OrtMemTypeDefault); + allocator.allocator_ = Ort::Allocator::Create(*allocator.session_, *memory_info); + } catch (const Ort::Exception& e) { + // WebGPU memory type name changed from "WebGPU_Buffer" to "WebGPU_Buf" in ORT 1.24.3. + // Try the old name before giving up. + if (type == DeviceType::WEBGPU) { + auto fallback_info = OrtMemoryInfo::Create("WebGPU_Buffer", OrtAllocatorType::OrtDeviceAllocator, 0, OrtMemType::OrtMemTypeDefault); + try { + allocator.allocator_ = Ort::Allocator::Create(*allocator.session_, *fallback_info); + } catch (const Ort::Exception& fallback_e) { + throw std::runtime_error( + "Failed to create allocator for WebGPU. " + "Primary name '" + + std::string(name) + "' error: " + std::string(e.what()) + + "; fallback 'WebGPU_Buffer' error: " + std::string(fallback_e.what())); + } + } else { + throw std::runtime_error("Failed to create allocator for " + std::string(name) + ": " + std::string(e.what())); + } + } if (!allocator.allocator_) { allocator = {}; // Reset everything just to be safe throw std::runtime_error("Unexpected failure to create device memory allocator for " + std::string(name)); @@ -1020,7 +558,7 @@ std::vector SessionInfo::GetOutputSymbolicShape(const std::string& Model::Model(std::unique_ptr config) : config_{std::move(config)} { CreateSessionOptions(); - EnsureDeviceOrtInit(*p_device_, *config_, arena_cfg_); + EnsureDeviceOrtInit(*p_device_, *config_); // Only CUDA, TRT-RTX, RyzenAI and DML does every input on the device // For WebGPU, use device memory only if graph capture is enabled, otherwise use CPU @@ -1050,7 +588,6 @@ Model::~Model() { allocator.session_.reset(); allocator.allocator_.reset(); session_options_.reset(); - arena_cfg_.reset(); // DML objects are globally scoped and launch background threads that retain hardware resources. // These threads persist beyond the lifetime of a Model, preventing proper cleanup and potentially causing deadlocks. // To avoid blocking driver threads, we explicitly destroy DML objects when the Model is destroyed. @@ -1189,7 +726,7 @@ void Model::CreateSessionOptionsFromConfig(const Config::SessionOptions& config_ auto session_device = SetProviderSessionOptions(session_options, config_session_options.providers, config_session_options.provider_options, is_primary_session_options, - disable_graph_capture, *config_, arena_cfg_); + *config_, disable_graph_capture); if (!p_device_) { p_device_ = session_device; @@ -1202,12 +739,12 @@ void Model::CreateSessionOptionsFromConfig(const Config::SessionOptions& config_ void Model::CreateSessionOptions() { session_options_ = OrtSessionOptions::Create(); - CreateSessionOptionsFromConfig(config_->model.decoder.session_options, *session_options_, true, false); + CreateSessionOptionsFromConfig(config_->model.decoder.session_options, *session_options_, true); for (auto& pipeline_model : config_->model.decoder.pipeline) { if (pipeline_model.session_options.has_value()) { auto emplaced = pipeline_session_options_.emplace(pipeline_model.model_id, OrtSessionOptions::Create()); - CreateSessionOptionsFromConfig(*pipeline_model.session_options, *emplaced.first->second, false, false); + CreateSessionOptionsFromConfig(*pipeline_model.session_options, *emplaced.first->second, false); } } diff --git a/src/models/model.h b/src/models/model.h index b3e5583362..dca333c4d0 100644 --- a/src/models/model.h +++ b/src/models/model.h @@ -162,7 +162,6 @@ struct Model : std::enable_shared_from_this, LeakChecked, External std::unique_ptr config_; std::unique_ptr session_options_; - std::unique_ptr arena_cfg_; DeviceInterface* p_device_{}; // The device we're running on (matches device_type_) used for things that work the same on all devices DeviceInterface* p_device_inputs_{}; // For some model inputs, the device might be the CPU device (all but KV cache currently for WebGPU and DML) @@ -179,7 +178,7 @@ struct Model : std::enable_shared_from_this, LeakChecked, External void CreateSessionOptionsFromConfig(const Config::SessionOptions& config_session_options, OrtSessionOptions& session_options, bool is_primary_session_options, - bool disable_graph_capture); + bool disable_graph_capture = false); std::map> pipeline_session_options_; }; diff --git a/src/models/multi_modal.cpp b/src/models/multi_modal.cpp index 81aa857c7f..15e1b9fa49 100644 --- a/src/models/multi_modal.cpp +++ b/src/models/multi_modal.cpp @@ -95,18 +95,18 @@ MultiModalLanguageModel::MultiModalLanguageModel(std::unique_ptr config, // The non-decoder models don't support graph capture because of control flow nodes, so disable graph capture for them if (vision) { vision_session_options_ = OrtSessionOptions::Create(); - CreateSessionOptionsFromConfig(config_->model.vision.session_options.has_value() ? config_->model.vision.session_options.value() : config_->model.decoder.session_options, *vision_session_options_, true, true); + CreateSessionOptionsFromConfig(config_->model.vision.session_options.has_value() ? config_->model.vision.session_options.value() : config_->model.decoder.session_options, *vision_session_options_, true, /*disable_graph_capture=*/true); vision_session_ = CreateSession(ort_env, config_->model.vision.filename, vision_session_options_.get()); } if (speech) { speech_session_options_ = OrtSessionOptions::Create(); - CreateSessionOptionsFromConfig(config_->model.speech.session_options.has_value() ? config_->model.speech.session_options.value() : config_->model.decoder.session_options, *speech_session_options_, true, true); + CreateSessionOptionsFromConfig(config_->model.speech.session_options.has_value() ? config_->model.speech.session_options.value() : config_->model.decoder.session_options, *speech_session_options_, true, /*disable_graph_capture=*/true); speech_session_ = CreateSession(ort_env, config_->model.speech.filename, speech_session_options_.get()); } embedding_session_options_ = OrtSessionOptions::Create(); - CreateSessionOptionsFromConfig(config_->model.embedding.session_options.has_value() ? config_->model.embedding.session_options.value() : config_->model.decoder.session_options, *embedding_session_options_, true, true); + CreateSessionOptionsFromConfig(config_->model.embedding.session_options.has_value() ? config_->model.embedding.session_options.value() : config_->model.decoder.session_options, *embedding_session_options_, true, /*disable_graph_capture=*/true); embedding_session_ = CreateSession(ort_env, config_->model.embedding.filename, embedding_session_options_.get()); decoder_session_ = CreateSession(ort_env, config_->model.decoder.filename, session_options_.get()); diff --git a/src/models/nemotron_speech.cpp b/src/models/nemotron_speech.cpp index 8b560ec062..b99afbb1a3 100644 --- a/src/models/nemotron_speech.cpp +++ b/src/models/nemotron_speech.cpp @@ -127,19 +127,19 @@ NemotronSpeechModel::NemotronSpeechModel(std::unique_ptr config, OrtEnv& if (config_->model.encoder.session_options.has_value()) { CreateSessionOptionsFromConfig(config_->model.encoder.session_options.value(), - *encoder_session_options_, true, false); + *encoder_session_options_, true); } else { CreateSessionOptionsFromConfig(config_->model.decoder.session_options, - *encoder_session_options_, true, false); + *encoder_session_options_, true); } CreateSessionOptionsFromConfig(config_->model.decoder.session_options, - *decoder_session_options_, true, false); + *decoder_session_options_, true); if (config_->model.joiner.session_options.has_value()) { CreateSessionOptionsFromConfig(config_->model.joiner.session_options.value(), - *joiner_session_options_, true, false); + *joiner_session_options_, true); } else { CreateSessionOptionsFromConfig(config_->model.decoder.session_options, - *joiner_session_options_, true, false); + *joiner_session_options_, true); } // Load the three ONNX models diff --git a/src/models/onnxruntime_api.h b/src/models/onnxruntime_api.h index fce33c9c56..d619ce680d 100644 --- a/src/models/onnxruntime_api.h +++ b/src/models/onnxruntime_api.h @@ -459,9 +459,20 @@ struct OrtStatus { Ort::Abstract make_abstract; }; +struct OrtHardwareDevice { + OrtHardwareDeviceType Type() const; + uint32_t VendorId() const; + uint32_t DeviceId() const; + std::string Vendor() const; + const OrtKeyValuePairs* Metadata() const; + + Ort::Abstract make_abstract; +}; + struct OrtEpDevice { std::string Name() const; std::string Vendor() const; + const OrtHardwareDevice* Device() const; Ort::Abstract make_abstract; }; diff --git a/src/models/onnxruntime_inline.h b/src/models/onnxruntime_inline.h index e6b10a6e24..2e453cfe6d 100644 --- a/src/models/onnxruntime_inline.h +++ b/src/models/onnxruntime_inline.h @@ -356,6 +356,31 @@ inline std::string OrtEpDevice::Vendor() const { return std::string(vendor); } +inline const OrtHardwareDevice* OrtEpDevice::Device() const { + return Ort::api->EpDevice_Device(this); +} + +inline OrtHardwareDeviceType OrtHardwareDevice::Type() const { + return Ort::api->HardwareDevice_Type(this); +} + +inline uint32_t OrtHardwareDevice::VendorId() const { + return Ort::api->HardwareDevice_VendorId(this); +} + +inline uint32_t OrtHardwareDevice::DeviceId() const { + return Ort::api->HardwareDevice_DeviceId(this); +} + +inline std::string OrtHardwareDevice::Vendor() const { + const char* vendor = Ort::api->HardwareDevice_Vendor(this); + return std::string(vendor); +} + +inline const OrtKeyValuePairs* OrtHardwareDevice::Metadata() const { + return Ort::api->HardwareDevice_Metadata(this); +} + inline void OrtCommonEnvInit(OrtEnv& v, _In_ const char* logid) { if (strcmp(logid, "onnxruntime-node") == 0) { Ort::ThrowOnError(Ort::api->SetLanguageProjection(&v, OrtLanguageProjection::ORT_PROJECTION_NODEJS)); diff --git a/src/models/session_options.cpp b/src/models/session_options.cpp new file mode 100644 index 0000000000..f2bef57aa5 --- /dev/null +++ b/src/models/session_options.cpp @@ -0,0 +1,202 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" + +#include +#include + +#include "../cuda/session_options.h" +#include "../dml/session_options.h" +#include "../nvtensorrtrtx/session_options.h" +#include "../openvino/session_options.h" +#include "../qnn/session_options.h" +#include "../rocm/session_options.h" +#include "../ryzenai/session_options.h" +#include "../vitisai/session_options.h" +#include "../webgpu/session_options.h" + +namespace Generators { + +// Each execution provider has a dedicated AppendExecutionProvider function in its +// own src//session_options.cpp file. The dispatch table in +// SetProviderSessionOptions (below) maps provider names to these functions. +// Providers that are not in the dispatch table are treated as generic and +// attempted via V2 (plugin) then V1 (legacy) API paths. + +std::vector ApplyDeviceFiltering(const Config::ProviderOptions& provider_options, + const std::vector& devices) { + const auto& filtering_options = provider_options.device_filtering_options; + if (!filtering_options || + (!filtering_options->hardware_device_id && + !filtering_options->hardware_vendor_id && + !filtering_options->hardware_device_type)) { + return devices; + } + + std::vector filtered_devices; + + for (const auto* device : devices) { + bool match = true; + auto* ort_device = device->Device(); + + if (filtering_options->hardware_device_id && + ort_device->DeviceId() != *filtering_options->hardware_device_id) { + match = false; + } + + if (filtering_options->hardware_vendor_id && + ort_device->VendorId() != *filtering_options->hardware_vendor_id) { + match = false; + } + + if (filtering_options->hardware_device_type && + ort_device->Type() != *filtering_options->hardware_device_type) { + match = false; + } + + if (match) { + filtered_devices.push_back(device); + } + } + + if (filtered_devices.empty()) { + std::string error_msg = "No devices matched the filtering criteria specified in provider options. Filter criteria:"; + if (filtering_options->hardware_device_id) { + error_msg += " hardware_device_id=" + std::to_string(*filtering_options->hardware_device_id); + } + if (filtering_options->hardware_vendor_id) { + error_msg += " hardware_vendor_id=" + std::to_string(*filtering_options->hardware_vendor_id); + } + if (filtering_options->hardware_device_type) { + error_msg += " hardware_device_type=" + std::to_string(static_cast(*filtering_options->hardware_device_type)); + } + error_msg += ". Available devices:"; + for (const auto* device : devices) { + auto* ort_device = device->Device(); + error_msg += " [device_id=" + std::to_string(ort_device->DeviceId()) + + ", vendor_id=" + std::to_string(ort_device->VendorId()) + + ", type=" + std::to_string(static_cast(ort_device->Type())) + "]"; + } + error_msg += ". Verify that the device filtering options in genai_config.json match an available device."; + throw std::runtime_error(error_msg); + } + return filtered_devices; +} + +// Helper to check if a provider is registered and get all matching EP devices +std::vector FindRegisteredEpDevices(const std::string& ep_name) { + auto device_ptrs = GetOrtEnv().GetEpDevices(); + std::vector ep_devices_ptrs; + for (auto* device : device_ptrs) { + if (device->Name() == ep_name) { + ep_devices_ptrs.push_back(device); + } + } + return ep_devices_ptrs; +} + +// If an execution provider is plugged-in via the registered plugin mechanism, +// this function appends the provider using the V2 API. +// Returns true if the provider was appended via the plugin path, false if the +// provider is not registered. +bool AppendExecutionProviderV2( + OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + DeviceType device_type, + const std::string& ep_name) { + auto ep_devices_ptrs = FindRegisteredEpDevices(ep_name); + if (ep_devices_ptrs.empty()) return false; // Not registered + + std::unordered_map options; + for (auto& option : provider_options.options) { + options.insert(option); + } + + auto filtered_ep_device_ptrs = ApplyDeviceFiltering(provider_options, ep_devices_ptrs); + if (device_type == DeviceType::WEBGPU) { + // WebGPU EP factory currently only supports one device at a time + filtered_ep_device_ptrs = {filtered_ep_device_ptrs.front()}; + } + + session_options.AppendExecutionProvider_V2(GetOrtEnv(), filtered_ep_device_ptrs, options); + return true; +} + +void AppendExecutionProviderV1(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options) { + std::vector keys, values; + for (auto& option : provider_options.options) { + keys.emplace_back(option.first.c_str()); + values.emplace_back(option.second.c_str()); + } + session_options.AppendExecutionProvider(provider_options.name.c_str(), keys.data(), + values.data(), keys.size()); +} + +DeviceInterface* SetProviderSessionOptions(OrtSessionOptions& session_options, + const std::vector& providers, + const std::vector& provider_options_list, + bool is_primary_session_options, + const Config& config, + bool disable_graph_capture) { + using AppendExecutionProviderFn = DeviceInterface* (*)(OrtSessionOptions&, + const Config::ProviderOptions&, + const Config&, + bool); + + // Dispatch table: maps provider name (as it appears in genai_config.json) to + // the corresponding provider-specific AppendExecutionProvider function. + static const std::unordered_map append_execution_provider{ + {"cuda", CUDAExecutionProvider::AppendExecutionProvider}, + {"DML", DMLExecutionProvider::AppendExecutionProvider}, + {"NvTensorRtRtx", NvTensorRtRtxExecutionProvider::AppendExecutionProvider}, + {"OpenVINO", OpenVINOExecutionProvider::AppendExecutionProvider}, + {"RyzenAI", RyzenAIExecutionProvider::AppendExecutionProvider}, + {"QNN", QNNExecutionProvider::AppendExecutionProvider}, + {"rocm", ROCmExecutionProvider::AppendExecutionProvider}, + {"VitisAI", VitisAIExecutionProvider::AppendExecutionProvider}, + {"WebGPU", WebGPUExecutionProvider::AppendExecutionProvider}, + }; + + DeviceInterface* device{}; + + auto providers_list = providers; + if (!is_primary_session_options) { + // Providers specified in a non-primary provider options list are added + // to the primary providers. They are considered immutable and implicitly + // added as providers. + for (const auto& provider_options : provider_options_list) { + if (std::find(providers_list.begin(), providers_list.end(), provider_options.name) == providers_list.end()) { + providers_list.push_back(provider_options.name); + } + } + } + + for (const auto& provider : providers_list) { + auto provider_options_it = std::find_if(provider_options_list.begin(), provider_options_list.end(), + [&provider](const Config::ProviderOptions& po) { return po.name == provider; }); + + if (provider_options_it == provider_options_list.end()) { + throw std::runtime_error("Provider options not found for provider: " + provider); + } + const auto& provider_options = *provider_options_it; + + const auto append_provider_it = append_execution_provider.find(provider_options.name); + if (append_provider_it != append_execution_provider.end()) { + auto session_device = append_provider_it->second(session_options, provider_options, config, disable_graph_capture); + if (is_primary_session_options && session_device && !device) { + device = session_device; // Set the device if not already set by a previous provider + } + } else { + if (!AppendExecutionProviderV2(session_options, provider_options, + DeviceType::CPU, provider_options.name)) { + AppendExecutionProviderV1(session_options, provider_options); + } + } + } + + return device; +} + +} // namespace Generators diff --git a/src/models/session_options.h b/src/models/session_options.h new file mode 100644 index 0000000000..af13172a17 --- /dev/null +++ b/src/models/session_options.h @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators { + +// Filters a list of EP devices according to the device_filtering_options specified +// in provider_options (hardware_device_id, hardware_vendor_id, hardware_device_type). +// Returns the full list unchanged if no filtering criteria are set. +// Throws std::runtime_error if criteria are set but no devices match. +std::vector ApplyDeviceFiltering(const Config::ProviderOptions& provider_options, + const std::vector& devices); + +// Returns all OrtEpDevice instances whose EP name matches |ep_name|. +// Returns an empty vector if the provider is not registered as a plugin. +std::vector FindRegisteredEpDevices(const std::string& ep_name); + +// Attempts to append an execution provider via the V2 plugin API. +// Discovers registered EP devices for |ep_name|, applies device filtering, and +// calls AppendExecutionProvider_V2. Returns true on success, false if the +// provider is not registered (caller should fall back to V1). +bool AppendExecutionProviderV2(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + DeviceType device_type, + const std::string& ep_name); + +// Appends an execution provider using the legacy V1 API (key/value string pairs). +void AppendExecutionProviderV1(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options); + +// Iterates over the requested providers, dispatches to provider-specific +// AppendExecutionProvider implementations, and returns the DeviceInterface +// for the first provider that supplies one (or nullptr if none do). +DeviceInterface* SetProviderSessionOptions(OrtSessionOptions& session_options, + const std::vector& providers, + const std::vector& provider_options_list, + bool is_primary_session_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators diff --git a/src/models/whisper.cpp b/src/models/whisper.cpp index 903b227c14..8e5e9fbc07 100644 --- a/src/models/whisper.cpp +++ b/src/models/whisper.cpp @@ -8,7 +8,7 @@ namespace Generators { WhisperModel::WhisperModel(std::unique_ptr config, OrtEnv& ort_env) : Model{std::move(config)} { encoder_session_options_ = OrtSessionOptions::Create(); - CreateSessionOptionsFromConfig(config_->model.encoder.session_options.has_value() ? config_->model.encoder.session_options.value() : config_->model.decoder.session_options, *encoder_session_options_, true, false); + CreateSessionOptionsFromConfig(config_->model.encoder.session_options.has_value() ? config_->model.encoder.session_options.value() : config_->model.decoder.session_options, *encoder_session_options_, true); session_encoder_ = CreateSession(ort_env, config_->model.encoder.filename, encoder_session_options_.get()); session_decoder_ = CreateSession(ort_env, config_->model.decoder.filename, session_options_.get()); diff --git a/src/nvtensorrtrtx/session_options.cpp b/src/nvtensorrtrtx/session_options.cpp new file mode 100644 index 0000000000..8461b91db6 --- /dev/null +++ b/src/nvtensorrtrtx/session_options.cpp @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" + +#include "../models/session_options.h" +#include "../cuda/session_options.h" +#include "../models/kv_cache.h" + +namespace Generators::NvTensorRtRtxExecutionProvider { + +namespace { + +void ConfigureProfile(const Config& config, OrtSessionOptions& session_options, bool is_multi_profile_enabled) { + // Get model parameters from decoder config + const int num_layers = config.model.decoder.num_hidden_layers; + const int num_kv_heads = config.model.decoder.num_key_value_heads; + const int head_dim = config.model.decoder.head_size; + const int batch_size = config.search.batch_size * config.search.num_beams; + + // Get max context length from config + const int max_context_len = config.model.context_length; + + // Extract KV cache name patterns from decoder config + std::string_view past_key_pattern = config.model.decoder.inputs.past_key_names; + std::string_view past_value_pattern = config.model.decoder.inputs.past_value_names; + + // Helper function to add KV cache with sequence length + const auto add_key_value_cache_shapes = [](std::ostringstream& shapes, + int batch_size, + std::string_view key_pattern, + std::string_view value_pattern, + int seq_len, + int num_layers, + int num_kv_heads, + int head_dim) { + for (int i = 0; i < num_layers; i++) { + // Use the existing function to format the key/value names + const std::string key_name = ComposeKeyValueName(std::string(key_pattern), i); + const std::string value_name = ComposeKeyValueName(std::string(value_pattern), i); + + shapes << "," << key_name << ":" << batch_size << "x" << num_kv_heads << "x" << seq_len << "x" << head_dim; + shapes << "," << value_name << ":" << batch_size << "x" << num_kv_heads << "x" << seq_len << "x" << head_dim; + } + }; + + if (is_multi_profile_enabled) { + // Multi-profile mode: existing logic for context and generation phases + const int opt_context_len = config.model.context_length / 2; + const int min_seq_len = 1; + + // Helper function to add input shapes (input_ids, attention_mask, position_ids) + const auto add_input_shapes = [](std::ostringstream& shapes, int batch_size, int seq_len, bool append = false) { + if (append) shapes << ","; + shapes << Config::Defaults::InputIdsName << ":" << batch_size << "x" << seq_len << "," + << Config::Defaults::AttentionMaskName << ":" << batch_size << "x" << seq_len; + }; + + // Helper function to add generation phase input shapes + const auto add_generation_input_shapes = [](std::ostringstream& shapes, int batch_size, int context_len) { + shapes << "," << Config::Defaults::AttentionMaskName << ":" << batch_size << "x" << context_len << "," + << Config::Defaults::InputIdsName << ":" << batch_size << "x1"; + }; + + // Helper function to add empty KV cache shapes for all layers + const auto add_empty_key_value_cache_shapes = [](std::ostringstream& shapes, + int batch_size, + std::string_view key_pattern, + std::string_view value_pattern, + int num_layers, + int num_kv_heads, + int head_dim) { + for (int i = 0; i < num_layers; i++) { + // Use the existing function to format the key/value names + const std::string key_name = ComposeKeyValueName(std::string(key_pattern), i); + const std::string value_name = ComposeKeyValueName(std::string(value_pattern), i); + + shapes << "," << key_name << ":" << batch_size << "x" << num_kv_heads << "x0x" << head_dim; + shapes << "," << value_name << ":" << batch_size << "x" << num_kv_heads << "x0x" << head_dim; + } + }; + + std::ostringstream min_shapes, opt_shapes, max_shapes; + + // MIN SHAPES (context phase and first token generation) + add_input_shapes(min_shapes, batch_size, min_seq_len); + add_empty_key_value_cache_shapes(min_shapes, batch_size, past_key_pattern, past_value_pattern, num_layers, num_kv_heads, head_dim); + add_generation_input_shapes(min_shapes, batch_size, min_seq_len); + add_key_value_cache_shapes(min_shapes, batch_size, past_key_pattern, past_value_pattern, min_seq_len, num_layers, num_kv_heads, head_dim); + + // OPT SHAPES (prefill with medium context and generation after medium context) + add_input_shapes(opt_shapes, batch_size, opt_context_len); + add_empty_key_value_cache_shapes(opt_shapes, batch_size, past_key_pattern, past_value_pattern, num_layers, num_kv_heads, head_dim); + add_generation_input_shapes(opt_shapes, batch_size, opt_context_len); + add_key_value_cache_shapes(opt_shapes, batch_size, past_key_pattern, past_value_pattern, opt_context_len - 1, num_layers, num_kv_heads, head_dim); + + // MAX SHAPES (prefill with maximum context and generation after maximum context) + add_input_shapes(max_shapes, batch_size, max_context_len); + add_key_value_cache_shapes(max_shapes, batch_size, past_key_pattern, past_value_pattern, max_context_len - 1, num_layers, num_kv_heads, head_dim); + add_generation_input_shapes(max_shapes, batch_size, max_context_len); + add_key_value_cache_shapes(max_shapes, batch_size, past_key_pattern, past_value_pattern, max_context_len - 1, num_layers, num_kv_heads, head_dim); + + // Add the constructed profiles to session options + session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_min_shapes", min_shapes.str().c_str()); + session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_opt_shapes", opt_shapes.str().c_str()); + session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_max_shapes", max_shapes.str().c_str()); + } else { + // Single profile mode: simple shapes with batch_dim=[1,1,batch_size] and seq_dim=[1,1024,max_context_len] + std::ostringstream min_shapes, opt_shapes, max_shapes; + + // MIN SHAPES: batch_dim=1, seq_dim=1 + constexpr int min_context_len = 1; + constexpr int min_batch_size = 1; + min_shapes << Config::Defaults::InputIdsName << ":" << min_batch_size << "x" << min_context_len << "," + << Config::Defaults::AttentionMaskName << ":" << min_batch_size << "x" << min_context_len; + add_key_value_cache_shapes(min_shapes, min_batch_size, past_key_pattern, past_value_pattern, 0, num_layers, num_kv_heads, head_dim); + + // OPT SHAPES: batch_dim=1, seq_dim=1024 + const int opt_context_len = std::min(max_context_len / 2, 1024); // Use a reasonable opt context length + constexpr int opt_batch_size = 1; // Use a opt batch size of 1 + // keeping seq length to 1 as optimizing for the gen phase + opt_shapes << Config::Defaults::InputIdsName << ":" << opt_batch_size << "x" << 1 << "," + << Config::Defaults::AttentionMaskName << ":" << opt_batch_size << "x" << opt_context_len; + add_key_value_cache_shapes(opt_shapes, opt_batch_size, past_key_pattern, past_value_pattern, opt_context_len, num_layers, num_kv_heads, head_dim); + + // MAX SHAPES: seq_dim=max_context_len + max_shapes << Config::Defaults::InputIdsName << ":" << batch_size << "x" << max_context_len << "," + << Config::Defaults::AttentionMaskName << ":" << batch_size << "x" << max_context_len; + add_key_value_cache_shapes(max_shapes, batch_size, past_key_pattern, past_value_pattern, max_context_len, num_layers, num_kv_heads, head_dim); + + // Add the constructed profiles to session options + session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_min_shapes", min_shapes.str().c_str()); + session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_opt_shapes", opt_shapes.str().c_str()); + session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.nv_profile_max_shapes", max_shapes.str().c_str()); + } +} + +} // namespace + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool /*disable_graph_capture*/) { + auto device = GetDeviceInterface(DeviceType::NvTensorRtRtx); + Generators::CUDAExecutionProvider::AddCudaStreamConfig(session_options, device); + Generators::CUDAExecutionProvider::AddCudaStreamConfig( + session_options, device, "ep.nvtensorrtrtxexecutionprovider.user_compute_stream"); + + // Configure NvTensorRT-specific settings (needed for both pre-registered and built-in paths) + NvTensorRtRtxExecutionProvider::ConfigureProfile(config, session_options, + IsMultiProfileEnabled(config.model.decoder.session_options)); + if (IsGraphCaptureEnabled(config.model.decoder.session_options)) { + session_options.AddConfigEntry("ep.nvtensorrtrtxexecutionprovider.enable_cuda_graph", "1"); + } + + // Try pre-registered plugin path first + if (!AppendExecutionProviderV2(session_options, provider_options, + DeviceType::NvTensorRtRtx, "NvTensorRTRTXExecutionProvider")) { + AppendExecutionProviderV1(session_options, provider_options); + } + + return device; +} + +} // namespace Generators::NvTensorRtRtxExecutionProvider diff --git a/src/nvtensorrtrtx/session_options.h b/src/nvtensorrtrtx/session_options.h new file mode 100644 index 0000000000..ee4cca7ae9 --- /dev/null +++ b/src/nvtensorrtrtx/session_options.h @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::NvTensorRtRtxExecutionProvider { + +// Configures NvTensorRtRtx profile shapes, sets the CUDA stream, and registers +// the execution provider on |session_options| (V2 plugin path, then V1 fallback). +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::NvTensorRtRtxExecutionProvider diff --git a/src/openvino/interface.cpp b/src/openvino/interface.cpp index b4324496b4..0eb49eb495 100644 --- a/src/openvino/interface.cpp +++ b/src/openvino/interface.cpp @@ -66,302 +66,4 @@ bool IsOpenVINOStatefulModel(const Model& model) { return false; } -static inline std::string GetOVDeviceStringFromOrtDevice(const OrtEpDevice* device_ptr) { - const OrtKeyValuePairs* keyvals = Ort::api->EpDevice_EpMetadata(device_ptr); - size_t num_entries; - const char* const* keys = nullptr; - const char* const* values = nullptr; - Ort::api->GetKeyValuePairs(keyvals, &keys, &values, &num_entries); - for (size_t kvi = 0; kvi < num_entries; kvi++) { - const std::string key = keys[kvi]; - const std::string val = values[kvi]; - if (key == "ov_device") { - return val; - } - } - - throw std::runtime_error("OrtEpDevice doesn't have ov_device meta field."); -} - -static inline const OrtEpDevice* SelectEpDeviceFromProviderOptions(const Generators::Config::ProviderOptions& provider_options) { - // Get device filtering config - Config::DeviceFilteringOptions resolved_device_filtering; - if (provider_options.device_filtering_options.has_value()) { - resolved_device_filtering = provider_options.device_filtering_options.value(); - } - - std::optional config_device_id = resolved_device_filtering.hardware_device_id; - std::optional config_vendor_id = resolved_device_filtering.hardware_vendor_id; - std::optional config_device_type_enum = resolved_device_filtering.hardware_device_type; - - // Use "device_type" in provider_options exclusively if it's provided - std::optional config_ov_device_type = std::nullopt; - - for (auto& option : provider_options.options) { - if (option.first == "device_type") { - config_ov_device_type = option.second; - } - } - - // if 'device_type' provider option has been set, this will take precedence over device_id/vendor_id/device_type_enum - if (config_ov_device_type.has_value()) { - config_device_id = std::nullopt; - config_vendor_id = std::nullopt; - config_device_type_enum = std::nullopt; - } - - // If config_ov_device_type isn't set, but there also hasn't been set any device_id/vendor_id/device_type_enum. - // In this case, default ov_device_type to "CPU". - if (!config_ov_device_type.has_value() && - !(config_device_id.has_value() || config_vendor_id.has_value() || config_device_type_enum.has_value())) { - config_ov_device_type = "CPU"; - } - - size_t num_devices = 0; - const OrtEpDevice* const* device_ptrs = nullptr; - Ort::GetEpDevices(&GetOrtEnv(), &device_ptrs, &num_devices); - - const std::string ep_name = "OpenVINOExecutionProvider"; - std::string chosen_ov_device; - for (size_t i = 0; i < num_devices; ++i) { - // skip this device if it's not an OpenVINO device. - if (Ort::api->EpDevice_EpName(device_ptrs[i]) != ep_name) - continue; - - const OrtHardwareDevice* hardware_device = Ort::api->EpDevice_Device(device_ptrs[i]); - const uint32_t hardware_device_id = Ort::api->HardwareDevice_DeviceId(hardware_device); - const uint32_t hardware_vendor_id = Ort::api->HardwareDevice_VendorId(hardware_device); - const OrtHardwareDeviceType hardware_device_type = Ort::api->HardwareDevice_Type(hardware_device); - - auto check_ov_device_type = [&config_ov_device_type, &provider_options](const OrtEpDevice* device_ptr) -> bool { - if (!config_ov_device_type.has_value()) { - return true; - } else { - auto meta_ov_device = GetOVDeviceStringFromOrtDevice(device_ptr); - if (meta_ov_device.find(*config_ov_device_type) != std::string::npos) { - return true; - } - return false; - } - }; - - bool hardware_device_id_matched = (!config_device_id.has_value()) || config_device_id.value() == hardware_device_id; - bool hardware_vendor_id_matched = (!config_vendor_id.has_value()) || config_vendor_id.value() == hardware_vendor_id; - bool hardware_device_type_matched = (!config_device_type_enum.has_value()) || - config_device_type_enum.value() == hardware_device_type; - bool hardware_ov_device_type_matched = check_ov_device_type(device_ptrs[i]); - // Append matched EP device - if (hardware_device_id_matched && - hardware_vendor_id_matched && - hardware_device_type_matched && - hardware_ov_device_type_matched) { - return device_ptrs[i]; - } - } - - return nullptr; -} - -static inline void EscapeBackslashes(std::string& s) { - size_t pos = 0; - while ((pos = s.find("\\", pos)) != std::string::npos) { - s.replace(pos, 1, "\\\\"); - pos += 2; - } -} - -static inline std::string MakeCacheDirAbsolute(std::string cache_dir, fs::path config_path) { - fs::path cache_dir_path(cache_dir); - - // if cache_dir is a relative path, then make it absolute. - if (cache_dir_path.is_relative()) { - fs::path abs_cache_dir = config_path / cache_dir_path; - std::string abs_cache_dir_str = abs_cache_dir.string(); - // convert '\' to '\\' - EscapeBackslashes(abs_cache_dir_str); - return abs_cache_dir_str; - } - - EscapeBackslashes(cache_dir); - return cache_dir; -} - -static inline void ReplaceCommaBrace(std::string& s) { - const std::string from = ",}"; - const std::string to = "}"; - - size_t pos = 0; - while ((pos = s.find(from, pos)) != std::string::npos) { - s.replace(pos, from.length(), to); - // No need to advance pos because replacement is shorter - } -} - -static inline void RemoveAllWhitespace(std::string& s) { - s.erase(std::remove_if(s.begin(), s.end(), [](unsigned char c) { return std::isspace(c); }), s.end()); -} - -static inline bool StartsWith(const std::string& str, const std::string& prefix) { - return str.size() >= prefix.size() && - str.compare(0, prefix.size(), prefix) == 0; -} - -static inline bool EndsWith(const std::string& str, const std::string& suffix) { - return str.size() >= suffix.size() && - str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; -} - -static inline std::optional AddCacheDirToLoadConfig(const std::string& cache_dir, - std::optional load_config_option, - const std::string& ov_device) { - // convert raw cache_dir path into OpenVINO key/value pair - std::string cache_dir_option = "\"CACHE_DIR\":\"" + cache_dir + "\""; - - // if load_config was set.. - if (load_config_option.has_value()) { - // load_config is set. We need to add the cache_dir OV option to the existing load_config. - auto& load_config_raw = *load_config_option; - - // few sanity checks.. - if (EndsWith(load_config_raw, ".json")) { - if (g_log.enabled) - Log("warning", "Unable to merge cache_dir into load_config when it references a .json file"); - return load_config_option; - } - - if (load_config_raw.find("CACHE_DIR") != std::string::npos) { - if (g_log.enabled) - Log("warning", "Unable to merge cache_dir into load_config, as it already defines CACHE_DIR"); - return load_config_option; - } - - // let's go ahead and try to merge in our load_config - // First, strip all whitespace to aid in any future pattern matching. - RemoveAllWhitespace(load_config_raw); - - if (!StartsWith(load_config_raw, "{")) { - if (g_log.enabled) - Log("warning", "Expected load_config to begin with '{'"); - return load_config_option; - } - - // first try to find the start of our device entry. For example, for CPU device, it would look like: - // "CPU":{ - std::string search_str = "\"" + ov_device + "\":{"; - size_t device_config_pos = load_config_raw.find(search_str); - if (device_config_pos != std::string::npos) { - // if it's found, we just want to insert our new CACHE_DIR option at the start of that - std::string replacement_string = search_str + cache_dir_option + ","; - load_config_raw.replace(device_config_pos, search_str.length(), replacement_string); - } else { - // there doesn't seem to be an entry for this device in the config. So, we'll just add one. - // Here, we'll find the first occurrence of '{' and replace it with '{"CPU":{"CACHE_DIR":""},' - size_t brace_pos = load_config_raw.find("{"); - if (brace_pos != std::string::npos) { - std::string replacement_string = "{" + search_str + cache_dir_option + "},"; - load_config_raw.replace(brace_pos, 1, replacement_string); - } - } - - // last step. In rare cases, it's possible that we added a ',' where we shouldn't have -- resulting in ',}' - // So replace ',}' with '}' - ReplaceCommaBrace(load_config_raw); - return load_config_raw; - } else { - // In this case, load_config hasn't been set. So it's pretty easy -- we just create one using - // the ov_device & cache_dir_option - load_config_option = "{\"" + ov_device + "\":{" + cache_dir_option + "}}"; - } - - return load_config_option; -} - -void OpenVINO_AppendProviderOptions(OrtSessionOptions& session_options, - const Generators::Config& config, - const Generators::Config::ProviderOptions& provider_options) { - if (provider_options.name != "OpenVINO") { - throw std::runtime_error("OpenVINO_AppendProviderOptions called with provider_options.name = " + provider_options.name); - } - -#if USE_WINML - // from the given provider options, select the right OVEP OrtDevice to use. - auto openvino_ep_device = SelectEpDeviceFromProviderOptions(provider_options); - if (!openvino_ep_device) { - throw std::runtime_error("OpenVINO_AppendProviderOptions: Unable to find suitable OpenVINOExecutionProvider OrtEpDevice"); - } - - // get the OpenVINO device string, from the selected device (e.g. "CPU", "GPU", "NPU", etc.) - auto selected_ov_device = GetOVDeviceStringFromOrtDevice(openvino_ep_device); - - std::vector keys, values; - std::optional cache_dir_option; - std::optional load_config_option; - for (auto& option : provider_options.options) { - // device type isn't a supported provider option when using SessionOptionsAppendExecutionProvider_V2 - // (It's set via the OrtDevice ptr which we selected above) - if (option.first == "device_type") { - continue; - } - - // For load_config we won't add to keys/vals just yet.. - if (option.first == "load_config") { - load_config_option = option.second; - continue; - } - - // For cache_dir, we will perform some manipulation and pack into load_config, - // so don't set it either. - if (option.first == "cache_dir") { - cache_dir_option = option.second; - continue; - } - - keys.emplace_back(option.first.c_str()); - values.emplace_back(option.second.c_str()); - } - - // if cache_dir option is set - if (cache_dir_option) { - // make it absolute - cache_dir_option = MakeCacheDirAbsolute(*cache_dir_option, config.config_path); - - // for SessionOptionsAppendExecutionProvider_V2, cache_dir isn't supported as a provider option, - // so add it to load_config. - load_config_option = AddCacheDirToLoadConfig(*cache_dir_option, load_config_option, selected_ov_device); - } - - if (load_config_option.has_value()) { - keys.emplace_back("load_config"); - values.emplace_back((*load_config_option).c_str()); - } - - std::vector ep_devices_ptrs = {openvino_ep_device}; - Ort::api->SessionOptionsAppendExecutionProvider_V2( - &session_options, - &GetOrtEnv(), - ep_devices_ptrs.data(), ep_devices_ptrs.size(), - keys.data(), values.data(), keys.size()); - -#else - std::vector keys, values; - std::optional cache_dir_option; - for (auto& option : provider_options.options) { - // For cache_dir, we will perform some manipulation before setting. - if (option.first == "cache_dir") { - cache_dir_option = option.second; - continue; - } - keys.emplace_back(option.first.c_str()); - values.emplace_back(option.second.c_str()); - } - - if (cache_dir_option) { - cache_dir_option = MakeCacheDirAbsolute(*cache_dir_option, config.config_path); - keys.emplace_back("cache_dir"); - values.emplace_back((*cache_dir_option).c_str()); - } - session_options.AppendExecutionProvider(provider_options.name.c_str(), keys.data(), values.data(), keys.size()); -#endif -} - } // namespace Generators diff --git a/src/openvino/session_options.cpp b/src/openvino/session_options.cpp new file mode 100644 index 0000000000..3cc84bd728 --- /dev/null +++ b/src/openvino/session_options.cpp @@ -0,0 +1,333 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" + +#include + +namespace Generators::OpenVINOExecutionProvider { + +static inline std::string GetOVDeviceStringFromOrtDevice(const OrtEpDevice* device_ptr) { + const OrtKeyValuePairs* keyvals = Ort::api->EpDevice_EpMetadata(device_ptr); + size_t num_entries; + const char* const* keys = nullptr; + const char* const* values = nullptr; + Ort::api->GetKeyValuePairs(keyvals, &keys, &values, &num_entries); + for (size_t kvi = 0; kvi < num_entries; kvi++) { + const std::string key = keys[kvi]; + const std::string val = values[kvi]; + if (key == "ov_device") { + return val; + } + } + + throw std::runtime_error("OrtEpDevice doesn't have ov_device meta field."); +} + +static inline const OrtEpDevice* SelectEpDeviceFromProviderOptions(const Generators::Config::ProviderOptions& provider_options) { + // Get device filtering config + Config::DeviceFilteringOptions resolved_device_filtering; + if (provider_options.device_filtering_options.has_value()) { + resolved_device_filtering = provider_options.device_filtering_options.value(); + } + + std::optional config_device_id = resolved_device_filtering.hardware_device_id; + std::optional config_vendor_id = resolved_device_filtering.hardware_vendor_id; + std::optional config_device_type_enum = resolved_device_filtering.hardware_device_type; + + // Use "device_type" in provider_options exclusively if it's provided + std::optional config_ov_device_type = std::nullopt; + + for (auto& option : provider_options.options) { + if (option.first == "device_type") { + config_ov_device_type = option.second; + } + } + + // if 'device_type' provider option has been set, this will take precedence over device_id/vendor_id/device_type_enum + if (config_ov_device_type.has_value()) { + config_device_id = std::nullopt; + config_vendor_id = std::nullopt; + config_device_type_enum = std::nullopt; + } + + // If config_ov_device_type isn't set, but there also hasn't been set any device_id/vendor_id/device_type_enum. + // In this case, default ov_device_type to "CPU". + if (!config_ov_device_type.has_value() && + !(config_device_id.has_value() || config_vendor_id.has_value() || config_device_type_enum.has_value())) { + config_ov_device_type = "CPU"; + } + + size_t num_devices = 0; + const OrtEpDevice* const* device_ptrs = nullptr; + Ort::GetEpDevices(&GetOrtEnv(), &device_ptrs, &num_devices); + + const std::string ep_name = "OpenVINOExecutionProvider"; + bool any_ov_device_matched = false; + for (size_t i = 0; i < num_devices; ++i) { + // skip this device if it's not an OpenVINO device. + if (Ort::api->EpDevice_EpName(device_ptrs[i]) != ep_name) + continue; + + any_ov_device_matched = true; + + const OrtHardwareDevice* hardware_device = Ort::api->EpDevice_Device(device_ptrs[i]); + const uint32_t hardware_device_id = Ort::api->HardwareDevice_DeviceId(hardware_device); + const uint32_t hardware_vendor_id = Ort::api->HardwareDevice_VendorId(hardware_device); + const OrtHardwareDeviceType hardware_device_type = Ort::api->HardwareDevice_Type(hardware_device); + + auto check_ov_device_type = [&config_ov_device_type](const OrtEpDevice* device_ptr) -> bool { + if (!config_ov_device_type.has_value()) { + return true; + } else { + auto meta_ov_device = GetOVDeviceStringFromOrtDevice(device_ptr); + if (meta_ov_device.find(*config_ov_device_type) != std::string::npos) { + return true; + } + return false; + } + }; + + bool hardware_device_id_matched = (!config_device_id.has_value()) || config_device_id.value() == hardware_device_id; + bool hardware_vendor_id_matched = (!config_vendor_id.has_value()) || config_vendor_id.value() == hardware_vendor_id; + bool hardware_device_type_matched = (!config_device_type_enum.has_value()) || + config_device_type_enum.value() == hardware_device_type; + bool hardware_ov_device_type_matched = check_ov_device_type(device_ptrs[i]); + // Append matched EP device + if (hardware_device_id_matched && + hardware_vendor_id_matched && + hardware_device_type_matched && + hardware_ov_device_type_matched) { + return device_ptrs[i]; + } + } + + // If any OpenVINO device was found but none matched the filtering criteria, throw an error. + // Otherwise, if no OpenVINO devices were found at all, return nullptr. Fallback + // to appending via provider bridge. + if (any_ov_device_matched) { + std::string error_msg = "No OpenVINO devices matched the filtering criteria specified in provider options. Filter criteria:"; + if (config_device_id.has_value()) { + error_msg += " hardware_device_id=" + std::to_string(*config_device_id); + } + if (config_vendor_id.has_value()) { + error_msg += " hardware_vendor_id=" + std::to_string(*config_vendor_id); + } + if (config_device_type_enum.has_value()) { + error_msg += " hardware_device_type=" + std::to_string(static_cast(*config_device_type_enum)); + } + if (config_ov_device_type.has_value()) { + error_msg += " device_type=" + *config_ov_device_type; + } + error_msg += ". Available OpenVINO devices:"; + for (size_t i = 0; i < num_devices; ++i) { + if (Ort::api->EpDevice_EpName(device_ptrs[i]) != ep_name) + continue; + const OrtHardwareDevice* hw = Ort::api->EpDevice_Device(device_ptrs[i]); + error_msg += " [device_id=" + std::to_string(Ort::api->HardwareDevice_DeviceId(hw)) + + ", vendor_id=" + std::to_string(Ort::api->HardwareDevice_VendorId(hw)) + + ", type=" + std::to_string(static_cast(Ort::api->HardwareDevice_Type(hw))) + + ", ov_device=" + GetOVDeviceStringFromOrtDevice(device_ptrs[i]) + "]"; + } + error_msg += ". Verify that the device filtering options in genai_config.json match an available device."; + throw std::runtime_error(error_msg); + } + return nullptr; +} + +static inline void EscapeBackslashes(std::string& s) { + size_t pos = 0; + while ((pos = s.find("\\", pos)) != std::string::npos) { + s.replace(pos, 1, "\\\\"); + pos += 2; + } +} + +static inline std::string MakeCacheDirAbsolute(std::string cache_dir, fs::path config_path) { + fs::path cache_dir_path(cache_dir); + + // if cache_dir is a relative path, then make it absolute. + if (cache_dir_path.is_relative()) { + fs::path abs_cache_dir = config_path / cache_dir_path; + std::string abs_cache_dir_str = abs_cache_dir.string(); + // convert '\' to '\\' + EscapeBackslashes(abs_cache_dir_str); + return abs_cache_dir_str; + } + + EscapeBackslashes(cache_dir); + return cache_dir; +} + +static inline void ReplaceCommaBrace(std::string& s) { + const std::string from = ",}"; + const std::string to = "}"; + + size_t pos = 0; + while ((pos = s.find(from, pos)) != std::string::npos) { + s.replace(pos, from.length(), to); + // No need to advance pos because replacement is shorter + } +} + +static inline void RemoveAllWhitespace(std::string& s) { + s.erase(std::remove_if(s.begin(), s.end(), [](unsigned char c) { return std::isspace(c); }), s.end()); +} + +static inline bool StartsWith(const std::string& str, const std::string& prefix) { + return str.size() >= prefix.size() && + str.compare(0, prefix.size(), prefix) == 0; +} + +static inline bool EndsWith(const std::string& str, const std::string& suffix) { + return str.size() >= suffix.size() && + str.compare(str.size() - suffix.size(), suffix.size(), suffix) == 0; +} + +static inline std::optional AddCacheDirToLoadConfig(const std::string& cache_dir, + std::optional load_config_option, + const std::string& ov_device) { + // convert raw cache_dir path into OpenVINO key/value pair + std::string cache_dir_option = "\"CACHE_DIR\":\"" + cache_dir + "\""; + + // if load_config was set.. + if (load_config_option.has_value()) { + // load_config is set. We need to add the cache_dir OV option to the existing load_config. + auto& load_config_raw = *load_config_option; + + // few sanity checks.. + if (EndsWith(load_config_raw, ".json")) { + if (g_log.enabled) + Log("warning", "Unable to merge cache_dir into load_config when it references a .json file"); + return load_config_option; + } + + if (load_config_raw.find("CACHE_DIR") != std::string::npos) { + if (g_log.enabled) + Log("warning", "Unable to merge cache_dir into load_config, as it already defines CACHE_DIR"); + return load_config_option; + } + + // let's go ahead and try to merge in our load_config + // First, strip all whitespace to aid in any future pattern matching. + RemoveAllWhitespace(load_config_raw); + + if (!StartsWith(load_config_raw, "{")) { + if (g_log.enabled) + Log("warning", "Expected load_config to begin with '{'"); + return load_config_option; + } + + // first try to find the start of our device entry. For example, for CPU device, it would look like: + // "CPU":{ + std::string search_str = "\"" + ov_device + "\":{"; + size_t device_config_pos = load_config_raw.find(search_str); + if (device_config_pos != std::string::npos) { + // if it's found, we just want to insert our new CACHE_DIR option at the start of that + std::string replacement_string = search_str + cache_dir_option + ","; + load_config_raw.replace(device_config_pos, search_str.length(), replacement_string); + } else { + // there doesn't seem to be an entry for this device in the config. So, we'll just add one. + // Here, we'll find the first occurrence of '{' and replace it with '{"CPU":{"CACHE_DIR":""},' + size_t brace_pos = load_config_raw.find("{"); + if (brace_pos != std::string::npos) { + std::string replacement_string = "{" + search_str + cache_dir_option + "},"; + load_config_raw.replace(brace_pos, 1, replacement_string); + } + } + + // last step. In rare cases, it's possible that we added a ',' where we shouldn't have -- resulting in ',}' + // So replace ',}' with '}' + ReplaceCommaBrace(load_config_raw); + return load_config_raw; + } else { + // In this case, load_config hasn't been set. So it's pretty easy -- we just create one using + // the ov_device & cache_dir_option + load_config_option = "{\"" + ov_device + "\":{" + cache_dir_option + "}}"; + } + + return load_config_option; +} + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool /*disable_graph_capture*/) { + auto device = GetDeviceInterface(DeviceType::OpenVINO); + if (provider_options.name != "OpenVINO") { + throw std::runtime_error("OpenVINOExecutionProvider::AppendExecutionProvider called with provider_options.name = " + provider_options.name); + } + + // from the given provider options, select the right OVEP OrtDevice to use. + auto openvino_ep_device = SelectEpDeviceFromProviderOptions(provider_options); + if (openvino_ep_device) { + // get the OpenVINO device string, from the selected device (e.g. "CPU", "GPU", "NPU", etc.) + auto selected_ov_device = GetOVDeviceStringFromOrtDevice(openvino_ep_device); + + std::unordered_map options; + std::optional cache_dir_option; + std::optional load_config_option; + for (auto& option : provider_options.options) { + // device type isn't a supported provider option when using SessionOptionsAppendExecutionProvider_V2 + // (It's set via the OrtDevice ptr which we selected above) + if (option.first == "device_type") { + continue; + } + + // For load_config we won't add to keys/vals just yet.. + if (option.first == "load_config") { + load_config_option = option.second; + continue; + } + + // For cache_dir, we will perform some manipulation and pack into load_config, + // so don't set it either. + if (option.first == "cache_dir") { + cache_dir_option = option.second; + continue; + } + + options.insert(option); + } + + // if cache_dir option is set + if (cache_dir_option) { + // make it absolute + cache_dir_option = MakeCacheDirAbsolute(*cache_dir_option, config.config_path); + + // for SessionOptionsAppendExecutionProvider_V2, cache_dir isn't supported as a provider option, + // so add it to load_config. + load_config_option = AddCacheDirToLoadConfig(*cache_dir_option, load_config_option, selected_ov_device); + } + + if (load_config_option.has_value()) { + options["load_config"] = *load_config_option; + } + + std::vector ep_devices_ptrs = {openvino_ep_device}; + session_options.AppendExecutionProvider_V2(GetOrtEnv(), ep_devices_ptrs, options); + } else { + std::vector keys, values; + std::optional cache_dir_option; + for (auto& option : provider_options.options) { + // For cache_dir, we will perform some manipulation before setting. + if (option.first == "cache_dir") { + cache_dir_option = option.second; + continue; + } + keys.emplace_back(option.first.c_str()); + values.emplace_back(option.second.c_str()); + } + + if (cache_dir_option) { + cache_dir_option = MakeCacheDirAbsolute(*cache_dir_option, config.config_path); + keys.emplace_back("cache_dir"); + values.emplace_back((*cache_dir_option).c_str()); + } + session_options.AppendExecutionProvider(provider_options.name.c_str(), keys.data(), values.data(), keys.size()); + } + + return device; +} + +} // namespace Generators::OpenVINOExecutionProvider diff --git a/src/openvino/session_options.h b/src/openvino/session_options.h new file mode 100644 index 0000000000..28802c81f2 --- /dev/null +++ b/src/openvino/session_options.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::OpenVINOExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::OpenVINOExecutionProvider diff --git a/src/qnn/session_options.cpp b/src/qnn/session_options.cpp new file mode 100644 index 0000000000..f118e4ba72 --- /dev/null +++ b/src/qnn/session_options.cpp @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" +#include "../models/session_options.h" + +namespace Generators::QNNExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool /*disable_graph_capture*/) { + DeviceInterface* device = nullptr; + session_options.AddConfigEntry("ep.share_ep_contexts", "1"); + if (const auto opt_it = std::find_if( + provider_options.options.begin(), provider_options.options.end(), + [](const auto& pair) { return pair.first == "enable_htp_shared_memory_allocator"; }); + opt_it != provider_options.options.end() && opt_it->second == "1") { + device = GetDeviceInterface(DeviceType::QNN); + } + // is_primary_session_options is set to false because the device is set based on + // the presence of the "enable_htp_shared_memory_allocator" option, + // not based on whether this is the primary session options or not. + if (!AppendExecutionProviderV2(session_options, provider_options, + DeviceType::QNN, "QNNExecutionProvider")) { + AppendExecutionProviderV1(session_options, provider_options); + } + + return device; +} + +} // namespace Generators::QNNExecutionProvider diff --git a/src/qnn/session_options.h b/src/qnn/session_options.h new file mode 100644 index 0000000000..cb4c18ab06 --- /dev/null +++ b/src/qnn/session_options.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::QNNExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::QNNExecutionProvider diff --git a/src/rocm/session_options.cpp b/src/rocm/session_options.cpp new file mode 100644 index 0000000000..8270dca3ec --- /dev/null +++ b/src/rocm/session_options.cpp @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" +#include "../models/session_options.h" + +namespace Generators::ROCmExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& /*config*/, + bool /*disable_graph_capture*/) { + OrtROCMProviderOptions ort_provider_options; + + std::vector keys, values; + for (auto& option : provider_options.options) { + keys.emplace_back(option.first.c_str()); + values.emplace_back(option.second.c_str()); + } + + Ort::ThrowOnError(Ort::api->UpdateROCMProviderOptions(&ort_provider_options, keys.data(), values.data(), keys.size())); + session_options.AppendExecutionProvider_ROCM(ort_provider_options); + + return nullptr; +} + +} // namespace Generators::ROCmExecutionProvider diff --git a/src/rocm/session_options.h b/src/rocm/session_options.h new file mode 100644 index 0000000000..d5bbea6e85 --- /dev/null +++ b/src/rocm/session_options.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::ROCmExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::ROCmExecutionProvider diff --git a/src/ryzenai/session_options.cpp b/src/ryzenai/session_options.cpp new file mode 100644 index 0000000000..3d6544429a --- /dev/null +++ b/src/ryzenai/session_options.cpp @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" +#include "interface.h" + +namespace Generators::RyzenAIExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool /*disable_graph_capture*/) { + auto device = GetDeviceInterface(DeviceType::RyzenAI); + session_options.AddConfigEntry("model_root", config.config_path.string().c_str()); + GetRyzenAIInterface()->SetupProvider(session_options, provider_options.options); + + return device; +} + +} // namespace Generators::RyzenAIExecutionProvider diff --git a/src/ryzenai/session_options.h b/src/ryzenai/session_options.h new file mode 100644 index 0000000000..908c5d09d3 --- /dev/null +++ b/src/ryzenai/session_options.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::RyzenAIExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::RyzenAIExecutionProvider diff --git a/src/vitisai/session_options.cpp b/src/vitisai/session_options.cpp new file mode 100644 index 0000000000..3034c17850 --- /dev/null +++ b/src/vitisai/session_options.cpp @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" + +#include "../models/session_options.h" + +namespace Generators::VitisAIExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool /*disable_graph_capture*/) { + // These session config entries need to be in place before the EP is appended + // so both the plugin and legacy append paths see the same configuration. + session_options.AddConfigEntry("session.inter_op.allow_spinning", "0"); + session_options.AddConfigEntry("session.intra_op.allow_spinning", "0"); + session_options.AddConfigEntry("model_root", config.config_path.string().c_str()); + + // VitisAI does not have a device type specific allocator, so we use CPU + // as the device type. + if (!AppendExecutionProviderV2(session_options, provider_options, + DeviceType::CPU, "VitisAIExecutionProvider")) { + AppendExecutionProviderV1(session_options, provider_options); + } + +#if defined(_WIN32) + if (const auto opt_it = std::find_if(provider_options.options.begin(), provider_options.options.end(), + [](const auto& pair) { return pair.first == "external_ep_libray"; }); + opt_it != provider_options.options.end()) { + auto lib_name = opt_it->second; + HMODULE lib = LoadLibrary(lib_name.c_str()); + if (!lib) { + throw std::runtime_error("Failed to load external EP library: " + lib_name); + } + // The library must remain loaded for the lifetime of the process since it + // provides the EP factory and custom ops used by the ORT session. + using CreateEpFactoriesFunc = void (*)(void*, const OrtApiBase*, void*, OrtEpFactory**, size_t, size_t*); + if (const auto func = reinterpret_cast( + GetProcAddress(lib, "CreateEpFactories"))) { + OrtEpFactory* factory = nullptr; + size_t num = 1; + + func(nullptr, OrtGetApiBase(), nullptr, &factory, num, &num); + } + fs::path custom_ops_lib_path(lib_name); + session_options.RegisterCustomOpsLibrary(custom_ops_lib_path.c_str()); + } +#endif // WIN32 + + return nullptr; +} + +} // namespace Generators::VitisAIExecutionProvider diff --git a/src/vitisai/session_options.h b/src/vitisai/session_options.h new file mode 100644 index 0000000000..a489a077e2 --- /dev/null +++ b/src/vitisai/session_options.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::VitisAIExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::VitisAIExecutionProvider diff --git a/src/webgpu/session_options.cpp b/src/webgpu/session_options.cpp new file mode 100644 index 0000000000..f03d6c2290 --- /dev/null +++ b/src/webgpu/session_options.cpp @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "session_options.h" +#include "../models/session_options.h" + +namespace Generators::WebGPUExecutionProvider { + +// Always retrieves the WebGPU device interface so the caller can use it for +// device memory allocations, regardless of whether the EP is registered as a +// plugin (V2) or via the legacy (V1) path. +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool /*disable_graph_capture*/) { + auto device = GetDeviceInterface(DeviceType::WEBGPU); + if (!AppendExecutionProviderV2(session_options, provider_options, + DeviceType::WEBGPU, "WebGpuExecutionProvider")) { + AppendExecutionProviderV1(session_options, provider_options); + } + + return device; +} + +} // namespace Generators::WebGPUExecutionProvider diff --git a/src/webgpu/session_options.h b/src/webgpu/session_options.h new file mode 100644 index 0000000000..fce60f0aa7 --- /dev/null +++ b/src/webgpu/session_options.h @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include "../generators.h" + +namespace Generators::WebGPUExecutionProvider { + +DeviceInterface* AppendExecutionProvider(OrtSessionOptions& session_options, + const Config::ProviderOptions& provider_options, + const Config& config, + bool disable_graph_capture = false); + +} // namespace Generators::WebGPUExecutionProvider