diff --git a/include/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_options.h b/include/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_options.h index c49f1cabb2926..f099246105199 100644 --- a/include/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_options.h +++ b/include/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_options.h @@ -9,7 +9,6 @@ struct OrtNvTensorRtRtxProviderOptions; typedef struct OrtNvTensorRtRtxProviderOptions OrtNvTensorRtRtxProviderOptions; - /** * @namespace onnxruntime::nv::provider_option_names @@ -28,21 +27,21 @@ typedef struct OrtNvTensorRtRtxProviderOptions OrtNvTensorRtRtxProviderOptions; * - `kONNXBytestreamSize`: Specifies the size of the ONNX bytestream. */ namespace onnxruntime { - namespace nv { - namespace provider_option_names { - constexpr const char* kDeviceId = "device_id"; - constexpr const char* kHasUserComputeStream = "has_user_compute_stream"; - constexpr const char* kUserComputeStream = "user_compute_stream"; - constexpr const char* kMaxWorkspaceSize = "nv_max_workspace_size"; - constexpr const char* kDumpSubgraphs = "nv_dump_subgraphs"; - constexpr const char* kDetailedBuildLog = "nv_detailed_build_log"; - constexpr const char* kProfilesMinShapes = "nv_profile_min_shapes"; - constexpr const char* kProfilesMaxShapes = "nv_profile_max_shapes"; - constexpr const char* kProfilesOptShapes = "nv_profile_opt_shapes"; - constexpr const char* kCudaGraphEnable = "nv_cuda_graph_enable"; - constexpr const char* kONNXBytestream = "nv_onnx_bytestream"; - constexpr const char* kONNXBytestreamSize = "nv_onnx_bytestream_size"; +namespace nv { +namespace provider_option_names { +constexpr const char* kDeviceId = "device_id"; +constexpr const char* kHasUserComputeStream = "has_user_compute_stream"; +constexpr const char* kUserComputeStream = "user_compute_stream"; +constexpr const char* kMaxWorkspaceSize = "nv_max_workspace_size"; +constexpr const char* kDumpSubgraphs = "nv_dump_subgraphs"; +constexpr const char* kDetailedBuildLog = "nv_detailed_build_log"; +constexpr const char* kProfilesMinShapes = "nv_profile_min_shapes"; +constexpr const char* kProfilesMaxShapes = "nv_profile_max_shapes"; +constexpr const char* kProfilesOptShapes = "nv_profile_opt_shapes"; +constexpr const char* kCudaGraphEnable = "nv_cuda_graph_enable"; +constexpr const char* kONNXBytestream = "nv_onnx_bytestream"; +constexpr const char* kONNXBytestreamSize = "nv_onnx_bytestream_size"; - } // namespace provider_option_names - } // namespace nv -} +} // namespace provider_option_names +} // namespace nv +} // namespace onnxruntime diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index f310652178b8d..1012aefa21846 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -4941,129 +4941,129 @@ struct OrtApi { * \snippet{doc} snippets.dox OrtStatus Return Value * * \since Version 1.22. - */ + */ /** * @brief Appends the NVIDIA GPU execution provider to the session options. - * - * This function allows the user to enable and configure the NVIDIA GPU - * execution provider for ONNX Runtime. By appending this provider, the + * + * This function allows the user to enable and configure the NVIDIA GPU + * execution provider for ONNX Runtime. By appending this provider, the * session can leverage NVIDIA GPUs for accelerated computation. - * - * @param options A pointer to the session options object to which the + * + * @param options A pointer to the session options object to which the * execution provider will be appended. - * @param device_id The ID of the GPU device to use. This is typically an - * integer representing the GPU index (e.g., 0 for the + * @param device_id The ID of the GPU device to use. This is typically an + * integer representing the GPU index (e.g., 0 for the * first GPU, 1 for the second GPU, etc.). - * + * * @return A status object indicating success or failure of the operation. - * Returns nullptr if the operation is successful. Otherwise, - * returns a pointer to an OrtStatus object containing error + * Returns nullptr if the operation is successful. Otherwise, + * returns a pointer to an OrtStatus object containing error * details. */ - /** \brief Appends the Nv execution provider to the specified session options. - * - * Appends the Nv (NVIDIA) execution provider to the session options, using settings - * specified in the provided OrtNvTensorRtRtxProviderOptions structure. - * - * \param[in] options A pointer to an ::OrtSessionOptions instance to which the Nv execution provider will be appended. - * \param[in] nv_options A pointer to an ::OrtNvTensorRtRtxProviderOptions instance containing configuration for the Nv execution provider. - * - * \snippet{doc} snippets.dox OrtStatus Return Value - * \since Version 1.21 - */ -ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_Nv_TensorRT_RTX, - _In_ OrtSessionOptions* options, - _In_ const OrtNvTensorRtRtxProviderOptions* nv_options); - -/** \brief Creates an Nv provider options object. -* -* Allocates and initializes an ::OrtNvTensorRtRtxProviderOptions structure. -* -* \param[out] out A pointer to a location that will receive the newly created ::OrtNvTensorRtRtxProviderOptions instance. -* -* \snippet{doc} snippets.dox OrtStatus Return Value -* \since Version 1.21 -*/ -ORT_API2_STATUS(CreateNvTensorRtRtxProviderOptions, - _Outptr_ OrtNvTensorRtRtxProviderOptions** out); - -/** \brief Updates the Nv provider options with the specified sets of key-value pairs. -* -* Iterates over arrays of keys and values, applying each pair to the ::OrtNvTensorRtRtxProviderOptions instance. -* -* \param[in,out] nv_options An ::OrtNvTensorRtRtxProviderOptions instance to be updated. -* \param[in] provider_options_keys Pointer to an array of `const char*` keys. -* \param[in] provider_options_values Pointer to an array of `const char*` values corresponding to the above keys. -* \param[in] num_keys The number of key-value pairs to be applied. -* -* \snippet{doc} snippets.dox OrtStatus Return Value -* \since Version 1.22 -*/ -ORT_API2_STATUS(UpdateNvTensorRtRtxProviderOptions, - _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - _In_ size_t num_keys); - -/** \brief Retrieves a serialized string representation of the Nv provider options. -* -* Serializes the current state of the ::OrtNvTensorRtRtxProviderOptions into a string for easy logging or debugging. -* -* \param[in] nv_options A pointer to the ::OrtNvTensorRtRtxProviderOptions instance to be serialized. -* \param[in,out] allocator An ::OrtAllocator to allocate the returned string. -* \param[out] ptr A pointer to the allocated string. The caller is responsible for freeing -* this memory by using the same allocator's Free method. -* -* \snippet{doc} snippets.dox OrtStatus Return Value -* \since Version 1.22 -*/ -ORT_API2_STATUS(GetNvTensorRtRtxProviderOptionsAsString, - _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, - _Inout_ OrtAllocator* allocator, - _Outptr_ char** ptr); - -/** \brief Releases the Nv provider options object. -* -* Frees the memory associated with an ::OrtNvTensorRtRtxProviderOptions instance and sets the pointer to NULL. -* -* \param[in] input A pointer to the ::OrtNvTensorRtRtxProviderOptions instance to be released. -*/ -void (ORT_API_CALL* ReleaseNvTensorRtRtxProviderOptions)(_Frees_ptr_opt_ OrtNvTensorRtRtxProviderOptions* input); - -/** \brief Updates a single key-value pair in the Nv provider options with a given value pointer. -* -* Updates a single entry in the ::OrtNvTensorRtRtxProviderOptions using the provided key and value pointer. -* This is useful for setting complex or non-string values that might not be trivially convertible to string types. -* -* \param[in,out] nv_options The ::OrtNvTensorRtRtxProviderOptions instance to update. -* \param[in] key A null-terminated string specifying which option to update. -* \param[in] value A pointer to the value to set for the corresponding key. -* -* \snippet{doc} snippets.dox OrtStatus Return Value -* \since Version 1.22 -*/ -ORT_API2_STATUS(UpdateNvTensorRtRtxProviderOptionsWithValue, - _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, - _In_ const char* key, - _In_ void* value); - -/** \brief Retrieves a pointer to the value stored for the specified key in the Nv provider options. -* -* Looks up the key in the ::OrtNvTensorRtRtxProviderOptions instance and returns the associated value pointer, -* if present. The type of the pointer must be known by the caller to cast or interpret correctly. -* -* \param[in] nv_options A pointer to the ::OrtNvTensorRtRtxProviderOptions instance. -* \param[in] key A null-terminated string specifying which key to look up. -* \param[out] ptr A pointer to a location that will receive the value pointer associated with the given key. -* -* \snippet{doc} snippets.dox OrtStatus Return Value -* \since Version 1.22 -*/ -ORT_API2_STATUS(GetNvTensorRtRtxProviderOptionsByName, - _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, - _In_ const char* key, - _Outptr_ void** ptr); + /** \brief Appends the Nv execution provider to the specified session options. + * + * Appends the Nv (NVIDIA) execution provider to the session options, using settings + * specified in the provided OrtNvTensorRtRtxProviderOptions structure. + * + * \param[in] options A pointer to an ::OrtSessionOptions instance to which the Nv execution provider will be appended. + * \param[in] nv_options A pointer to an ::OrtNvTensorRtRtxProviderOptions instance containing configuration for the Nv execution provider. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.21 + */ + ORT_API2_STATUS(SessionOptionsAppendExecutionProvider_Nv_TensorRT_RTX, + _In_ OrtSessionOptions* options, + _In_ const OrtNvTensorRtRtxProviderOptions* nv_options); + + /** \brief Creates an Nv provider options object. + * + * Allocates and initializes an ::OrtNvTensorRtRtxProviderOptions structure. + * + * \param[out] out A pointer to a location that will receive the newly created ::OrtNvTensorRtRtxProviderOptions instance. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.21 + */ + ORT_API2_STATUS(CreateNvTensorRtRtxProviderOptions, + _Outptr_ OrtNvTensorRtRtxProviderOptions** out); + + /** \brief Updates the Nv provider options with the specified sets of key-value pairs. + * + * Iterates over arrays of keys and values, applying each pair to the ::OrtNvTensorRtRtxProviderOptions instance. + * + * \param[in,out] nv_options An ::OrtNvTensorRtRtxProviderOptions instance to be updated. + * \param[in] provider_options_keys Pointer to an array of `const char*` keys. + * \param[in] provider_options_values Pointer to an array of `const char*` values corresponding to the above keys. + * \param[in] num_keys The number of key-value pairs to be applied. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22 + */ + ORT_API2_STATUS(UpdateNvTensorRtRtxProviderOptions, + _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + _In_ size_t num_keys); + + /** \brief Retrieves a serialized string representation of the Nv provider options. + * + * Serializes the current state of the ::OrtNvTensorRtRtxProviderOptions into a string for easy logging or debugging. + * + * \param[in] nv_options A pointer to the ::OrtNvTensorRtRtxProviderOptions instance to be serialized. + * \param[in,out] allocator An ::OrtAllocator to allocate the returned string. + * \param[out] ptr A pointer to the allocated string. The caller is responsible for freeing + * this memory by using the same allocator's Free method. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22 + */ + ORT_API2_STATUS(GetNvTensorRtRtxProviderOptionsAsString, + _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, + _Inout_ OrtAllocator* allocator, + _Outptr_ char** ptr); + + /** \brief Releases the Nv provider options object. + * + * Frees the memory associated with an ::OrtNvTensorRtRtxProviderOptions instance and sets the pointer to NULL. + * + * \param[in] input A pointer to the ::OrtNvTensorRtRtxProviderOptions instance to be released. + */ + void(ORT_API_CALL* ReleaseNvTensorRtRtxProviderOptions)(_Frees_ptr_opt_ OrtNvTensorRtRtxProviderOptions* input); + + /** \brief Updates a single key-value pair in the Nv provider options with a given value pointer. + * + * Updates a single entry in the ::OrtNvTensorRtRtxProviderOptions using the provided key and value pointer. + * This is useful for setting complex or non-string values that might not be trivially convertible to string types. + * + * \param[in,out] nv_options The ::OrtNvTensorRtRtxProviderOptions instance to update. + * \param[in] key A null-terminated string specifying which option to update. + * \param[in] value A pointer to the value to set for the corresponding key. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22 + */ + ORT_API2_STATUS(UpdateNvTensorRtRtxProviderOptionsWithValue, + _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, + _In_ const char* key, + _In_ void* value); + + /** \brief Retrieves a pointer to the value stored for the specified key in the Nv provider options. + * + * Looks up the key in the ::OrtNvTensorRtRtxProviderOptions instance and returns the associated value pointer, + * if present. The type of the pointer must be known by the caller to cast or interpret correctly. + * + * \param[in] nv_options A pointer to the ::OrtNvTensorRtRtxProviderOptions instance. + * \param[in] key A null-terminated string specifying which key to look up. + * \param[out] ptr A pointer to a location that will receive the value pointer associated with the given key. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * \since Version 1.22 + */ + ORT_API2_STATUS(GetNvTensorRtRtxProviderOptionsByName, + _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, + _In_ const char* key, + _Outptr_ void** ptr); }; /* diff --git a/onnxruntime/core/providers/get_execution_providers.cc b/onnxruntime/core/providers/get_execution_providers.cc index 16cddc6cf4fae..9ecabcad504b3 100644 --- a/onnxruntime/core/providers/get_execution_providers.cc +++ b/onnxruntime/core/providers/get_execution_providers.cc @@ -23,9 +23,9 @@ constexpr ProviderInfo kProvidersInPriorityOrder[] = { kNvTensorRTRTXExecutionProvider, #ifdef USE_NV - true, + true, #else - false, + false, #endif }, { diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc index 7961e6ff44842..020d4be56580e 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.cc @@ -2108,7 +2108,7 @@ std::unique_ptr NvExecutionProvider::GetSubGraph(SubGraph_t gra } SubGraphCollection_t NvExecutionProvider::GetSupportedList(SubGraphCollection_t nodes_vector_input, int iterations, const int max_iterations, - const GraphViewer& graph, bool* early_termination) const { + const GraphViewer& graph, bool* early_termination) const { // Return if iterations are exceeding predefined number SubGraphCollection_t nodes_list_output; if (iterations > max_iterations) { @@ -2462,9 +2462,9 @@ bool NvExecutionProvider::DetectTensorRTGraphCycles(SubGraphCollection_t& suppor std::vector> NvExecutionProvider::GetCapability(const GraphViewer& graph, - const IKernelLookup& /*kernel_lookup*/, - const GraphOptimizerRegistry& /*graph_optimizer_registry*/, - IResourceAccountant* /* resource_accountant */) const { + const IKernelLookup& /*kernel_lookup*/, + const GraphOptimizerRegistry& /*graph_optimizer_registry*/, + IResourceAccountant* /* resource_accountant */) const { // Construct subgraph capability from node list std::vector> result; // Get ModelPath @@ -2697,14 +2697,14 @@ NvExecutionProvider::GetCapability(const GraphViewer& graph, * Refit the weight-stripped engine */ common::Status NvExecutionProvider::RefitEngine(std::string onnx_model_filename, - std::string& onnx_model_folder_path, - std::string& weight_stripped_engine_cath_path, - bool path_check, - const void* onnx_model_bytestream, - size_t onnx_model_bytestream_size, - nvinfer1::ICudaEngine* trt_engine, - bool serialize_refitted_engine, - bool detailed_build_log) { + std::string& onnx_model_folder_path, + std::string& weight_stripped_engine_cath_path, + bool path_check, + const void* onnx_model_bytestream, + size_t onnx_model_bytestream_size, + nvinfer1::ICudaEngine* trt_engine, + bool serialize_refitted_engine, + bool detailed_build_log) { #if NV_TENSORRT_MAJOR >= 10 bool refit_from_file = onnx_model_bytestream == nullptr && onnx_model_bytestream_size == 0; std::filesystem::path onnx_model_path{onnx_model_folder_path}; @@ -2778,7 +2778,7 @@ common::Status NvExecutionProvider::RefitEngine(std::string onnx_model_filename, } common::Status NvExecutionProvider::Compile(const std::vector& fused_nodes_and_graphs, - std::vector& node_compute_funcs) { + std::vector& node_compute_funcs) { for (auto& fused_node_graph : fused_nodes_and_graphs) { const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph; const Node& fused_node = fused_node_graph.fused_node; @@ -2816,10 +2816,10 @@ common::Status NvExecutionProvider::Compile(const std::vector } Status NvExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphViewer& graph_body_viewer, - const Node& fused_node, - std::unordered_map& input_map, - std::unordered_map& output_map, - std::vector& node_compute_funcs) { + const Node& fused_node, + std::unordered_map& input_map, + std::unordered_map& output_map, + std::vector& node_compute_funcs) { // Reconstruct graph proto from fused node's function body auto model = graph_body_viewer.CreateModel(*GetLogger()); auto model_proto = model->ToProto(); @@ -4054,10 +4054,10 @@ Status NvExecutionProvider::CreateNodeComputeInfoFromGraph(const GraphViewer& gr } Status NvExecutionProvider::CreateNodeComputeInfoFromPrecompiledEngine(const GraphViewer& graph_body_viewer, - const Node& fused_node, - std::unordered_map& input_map, - std::unordered_map& output_map, - std::vector& node_compute_funcs) { + const Node& fused_node, + std::unordered_map& input_map, + std::unordered_map& output_map, + std::vector& node_compute_funcs) { std::unique_ptr trt_engine; std::unique_ptr trt_context; std::unordered_map input_indexes; // TRT engine input name -> ORT kernel context input index diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.h b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.h index 6b7e04ac7cfae..4b0b44def77d0 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.h +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider.h @@ -248,9 +248,9 @@ class NvExecutionProvider : public IExecutionProvider { std::vector> GetCapability(const GraphViewer& graph, - const IKernelLookup& /*kernel_lookup*/, - const GraphOptimizerRegistry& graph_optimizer_registry, - IResourceAccountant* /* resource_accountant */) const override; + const IKernelLookup& /*kernel_lookup*/, + const GraphOptimizerRegistry& graph_optimizer_registry, + IResourceAccountant* /* resource_accountant */) const override; int GetDeviceId() const { return device_id_; } diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc index b344f2ed57618..5559e2e791d40 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc @@ -79,7 +79,7 @@ common::Status CreateTensorRTCustomOpDomainList(std::vector& bool (*dyn_initLibNvInferPlugins)(void* logger, char const* libNamespace); ORT_THROW_IF_ERROR(env.GetSymbolFromLibrary(library_handle, "initLibNvInferPlugins", (void**)&dyn_initLibNvInferPlugins)); dyn_initLibNvInferPlugins(&trt_logger, ""); - LOGS_DEFAULT(INFO) << "[Nv EP] Default plugins successfully loaded.";; + LOGS_DEFAULT(INFO) << "[Nv EP] Default plugins successfully loaded."; #if defined(_MSC_VER) #pragma warning(push) diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_helper.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_helper.cc index 10b00a33a85cc..5373b6fd08afc 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_helper.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_helper.cc @@ -36,8 +36,8 @@ std::string GetUniqueGraphName(const Graph& graph) { // The newly-built graph has not yet being resolved by Graph::Resolve(), so we can't leverage // Graph::ResolveContext::IsInputInitializerOrOutput(). We have to implement this fuction again. bool NvExecutionProvider::IsInputInitializerOrOutput(const Graph& graph, - const std::string& name, - bool check_ancestors) const { + const std::string& name, + bool check_ancestors) const { const Graph* parent_graph = nullptr; return IsLocalValue(graph, name) || (check_ancestors && (parent_graph = graph.ParentGraph()) != nullptr && @@ -47,7 +47,7 @@ bool NvExecutionProvider::IsInputInitializerOrOutput(const Graph& graph, // The newly-built graph has not yet being resolved by Graph::Resolve(), so we can't leverage // Graph::ResolveContext::IsOuterScopeValue(). We have to implement this function again. bool NvExecutionProvider::IsOuterScopeValue(const Graph& graph, - const std::string& name) const { + const std::string& name) const { const Graph* parent_graph = nullptr; return (parent_graph = graph.ParentGraph()) != nullptr && IsInputInitializerOrOutput(*parent_graph, name, true); @@ -56,7 +56,7 @@ bool NvExecutionProvider::IsOuterScopeValue(const Graph& graph, // The newly-built graph has not yet being resolved by Graph::Resolve(), so we can't leverage // Graph::ResolveContext::IsLocalValue(). We have to implement this function again. bool NvExecutionProvider::IsLocalValue(const Graph& graph, - const std::string& name) const { + const std::string& name) const { std::string unique_graph_name = GetUniqueGraphName(graph); if (subgraph_context_map_.find(unique_graph_name) == subgraph_context_map_.end()) { return false; @@ -127,7 +127,7 @@ void NvExecutionProvider::BuildSubGraphContext(const Graph& graph) const { // Set outer scope values for subgraphs and add thoes values as top-level graph's inputs if needed. void NvExecutionProvider::SetGraphOuterScopeValuesAndInputs(Graph& graph_build, - const Graph& graph) const { + const Graph& graph) const { // Iterate all the nodes and recurse into inner most subgraph first for both newly built graph and original graph for (int i = 0; i < graph_build.MaxNodeIndex(); ++i) { auto graph_build_node = graph_build.GetNode(i); diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_info.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_info.cc index 9f683c14bbddb..f625d7e8093b8 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_info.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_info.cc @@ -76,7 +76,7 @@ ProviderOptions NvExecutionProviderInfo::ToProviderOptions(const NvExecutionProv {nv::provider_option_names::kCudaGraphEnable, MakeStringWithClassicLocale(info.cuda_graph_enable)}, {nv::provider_option_names::kONNXBytestream, MakeStringWithClassicLocale(info.onnx_bytestream)}, {nv::provider_option_names::kONNXBytestreamSize, MakeStringWithClassicLocale(info.onnx_bytestream_size)}, - }; + }; return options; } diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc index 7effe287e56d0..be794bc369664 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.cc @@ -50,39 +50,36 @@ struct ProviderInfo_Nv_Impl final : ProviderInfo_Nv { if (!out) { return CreateStatus(ORT_INVALID_ARGUMENT, "Output pointer 'out' is NULL."); } - *out = NULL; // Initialize output + *out = NULL; // Initialize output struct OrtNvTensorRtRtxProviderOptions* options = (struct OrtNvTensorRtRtxProviderOptions*)malloc(sizeof(struct OrtNvTensorRtRtxProviderOptions)); if (!options) { - return CreateStatus(ORT_FAIL, "Failed to allocate memory for NvProviderOptions."); + return CreateStatus(ORT_FAIL, "Failed to allocate memory for NvProviderOptions."); } - options->magic_number = ORT_NV_PROVIDER_OPTIONS_MAGIC; // Set the magic number + options->magic_number = ORT_NV_PROVIDER_OPTIONS_MAGIC; // Set the magic number - *out = options; // Assign the created handle to the output parameter + *out = options; // Assign the created handle to the output parameter return nullptr; } void ReleaseProviderOptions(_Frees_ptr_opt_ OrtNvTensorRtRtxProviderOptions* options) override { if (options) { if (options->magic_number == ORT_NV_PROVIDER_OPTIONS_MAGIC) { - // If you had members that were heap-allocated *internally* by the options struct, free them here. - // Example: free(options->some_string_option); + // If you had members that were heap-allocated *internally* by the options struct, free them here. + // Example: free(options->some_string_option); - // Invalidate the magic number to help detect use-after-free issues. - options->magic_number = 0; + // Invalidate the magic number to help detect use-after-free issues. + options->magic_number = 0; - // Free the main struct allocation - free(options); + // Free the main struct allocation + free(options); } else { - // Handle error: Log a warning? Abort in debug mode? - // Attempting to release an invalid handle. - // Depending on policy, you might just ignore it or log defensively. + // Handle error: Log a warning? Abort in debug mode? + // Attempting to release an invalid handle. + // Depending on policy, you might just ignore it or log defensively. } + } } - } - - - } g_info; @@ -91,6 +88,8 @@ struct NvProviderFactory : IExecutionProviderFactory { ~NvProviderFactory() override {} std::unique_ptr CreateProvider() override; + std::unique_ptr CreateProvider(const OrtSessionOptions& session_options, + const OrtLogger& session_logger); private: NvExecutionProviderInfo info_; @@ -100,6 +99,35 @@ std::unique_ptr NvProviderFactory::CreateProvider() { return std::make_unique(info_); } +std::unique_ptr NvProviderFactory::CreateProvider(const OrtSessionOptions& session_options, const OrtLogger& session_logger) { + const ConfigOptions& config_options = session_options.GetConfigOptions(); + const std::unordered_map& config_options_map = config_options.GetConfigOptionsMap(); + + // The implementation of the SessionOptionsAppendExecutionProvider C API function automatically adds EP options to + // the session option configurations with the key prefix "ep..". + // We extract those EP options to create a new "provider options" key/value map. + std::string lowercase_ep_name = kNvTensorRTRTXExecutionProvider; + std::transform(lowercase_ep_name.begin(), lowercase_ep_name.end(), lowercase_ep_name.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + + ProviderOptions provider_options; + std::string key_prefix = "ep."; + key_prefix += lowercase_ep_name; + key_prefix += "."; + + for (const auto& [key, value] : config_options_map) { + if (key.rfind(key_prefix, 0) == 0) { + provider_options[key.substr(key_prefix.size())] = value; + } + } + NvExecutionProviderInfo info = onnxruntime::NvExecutionProviderInfo::FromProviderOptions(provider_options); + + auto ep = std::make_unique(info); + ep->SetLogger(reinterpret_cast(&session_logger)); + return ep; +} + struct Nv_Provider : Provider { void* GetInfo() override { return &g_info; } std::shared_ptr CreateExecutionProviderFactory(int device_id) override { diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.h b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.h index 3740520689a8a..83e144e76787c 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.h +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_factory.h @@ -11,8 +11,7 @@ struct ProviderInfo_Nv { virtual OrtStatus* GetTensorRTCustomOpDomainList(std::vector& domain_list, const std::string extra_plugin_lib_paths) = 0; virtual OrtStatus* ReleaseCustomOpDomainList(std::vector& domain_list) = 0; virtual OrtStatus* CreateProviderOptions(_Outptr_ OrtNvTensorRtRtxProviderOptions** out) = 0; - virtual void ReleaseProviderOptions(_Frees_ptr_opt_ OrtNvTensorRtRtxProviderOptions* options) = 0; - + virtual void ReleaseProviderOptions(_Frees_ptr_opt_ OrtNvTensorRtRtxProviderOptions* options) = 0; protected: ~ProviderInfo_Nv() = default; // Can only be destroyed through a subclass instance diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_options_internal.h b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_options_internal.h index ce2fcaba0a891..f68727587c62e 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_options_internal.h +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_provider_options_internal.h @@ -1,15 +1,15 @@ #ifndef ONNXRUNTIME_NV_PROVIDER_OPTIONS_INTERNAL_H #define ONNXRUNTIME_NV_PROVIDER_OPTIONS_INTERNAL_H -#include "core/providers/nv_tensorrt_rtx/nv_provider_options.h" // Include the public header first +#include "core/providers/nv_tensorrt_rtx/nv_provider_options.h" // Include the public header first // Include necessary standard headers -#include // For size_t -#include // For fixed-width types like uint32_t -#include // For bool type +#include // For size_t +#include // For fixed-width types like uint32_t +#include // For bool type // Define a magic number to help verify the handle validity at runtime -#define ORT_NV_PROVIDER_OPTIONS_MAGIC 0x4E56504F // ASCII for "NVPO" +#define ORT_NV_PROVIDER_OPTIONS_MAGIC 0x4E56504F // ASCII for "NVPO" /** * \brief Internal structure definition for OrtNvTensorRtRtxProviderOptions. @@ -18,28 +18,28 @@ * It contains the actual configuration data for the Nv provider. */ struct OrtNvTensorRtRtxProviderOptions { - // Magic number for basic type validation in API calls - uint32_t magic_number; - - // --- Configuration Members --- - - int device_id; // CUDA device ID. - bool has_user_compute_stream; // Indicator if user specified a CUDA compute stream. (Internal flag derived from user_compute_stream != NULL) - void* user_compute_stream; // User specified CUDA compute stream. NULL if not set. - bool cuda_graph_enable; // Enable CUDA graph capture. - size_t max_workspace_size; // Maximum workspace size in bytes. 0 means implementation defined default. - bool dump_subgraphs; // Dump EP subgraphs. - bool detailed_build_log; // Enable detailed TensorRT engine build logging. - - // --- TensorRT Profile Shapes --- - const char* profile_min_shapes; // Min shapes string. - const char* profile_max_shapes; // Max shapes string. - const char* profile_opt_shapes; // Optimal shapes string. - - // --- TensorRT ONNX Byte Stream for Weights --- - // This pointer points to memory owned by the USER. - const void* onnx_bytestream; // Pointer to original ONNX model byte stream. - size_t onnx_bytestream_size; // Size of the byte stream. + // Magic number for basic type validation in API calls + uint32_t magic_number; + + // --- Configuration Members --- + + int device_id; // CUDA device ID. + bool has_user_compute_stream; // Indicator if user specified a CUDA compute stream. (Internal flag derived from user_compute_stream != NULL) + void* user_compute_stream; // User specified CUDA compute stream. NULL if not set. + bool cuda_graph_enable; // Enable CUDA graph capture. + size_t max_workspace_size; // Maximum workspace size in bytes. 0 means implementation defined default. + bool dump_subgraphs; // Dump EP subgraphs. + bool detailed_build_log; // Enable detailed TensorRT engine build logging. + + // --- TensorRT Profile Shapes --- + const char* profile_min_shapes; // Min shapes string. + const char* profile_max_shapes; // Max shapes string. + const char* profile_opt_shapes; // Optimal shapes string. + + // --- TensorRT ONNX Byte Stream for Weights --- + // This pointer points to memory owned by the USER. + const void* onnx_bytestream; // Pointer to original ONNX model byte stream. + size_t onnx_bytestream_size; // Size of the byte stream. }; -#endif // ONNXRUNTIME_NV_PROVIDER_OPTIONS_INTERNAL_H +#endif // ONNXRUNTIME_NV_PROVIDER_OPTIONS_INTERNAL_H diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/onnx_ctx_model_helper.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/onnx_ctx_model_helper.cc index 57b27ccc116f9..b3c13c1d2e8ef 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/onnx_ctx_model_helper.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/onnx_ctx_model_helper.cc @@ -293,14 +293,14 @@ Status TensorRTCacheModelHandler::GetEpContextFromGraph(const GraphViewer& graph const std::string onnx_model_filename = attrs.at(ONNX_MODEL_FILENAME).s(); std::string placeholder; auto status = NvExecutionProvider::RefitEngine(onnx_model_filename, - onnx_model_folder_path_, - placeholder, - make_secure_path_checks, - onnx_model_bytestream_, - onnx_model_bytestream_size_, - (*trt_engine_).get(), - false /* serialize refitted engine to disk */, - detailed_build_log_); + onnx_model_folder_path_, + placeholder, + make_secure_path_checks, + onnx_model_bytestream_, + onnx_model_bytestream_size_, + (*trt_engine_).get(), + false /* serialize refitted engine to disk */, + detailed_build_log_); if (status != Status::OK()) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, status.ErrorMessage()); } @@ -362,14 +362,14 @@ Status TensorRTCacheModelHandler::GetEpContextFromGraph(const GraphViewer& graph const std::string onnx_model_filename = attrs.at(ONNX_MODEL_FILENAME).s(); std::string weight_stripped_engine_cache = engine_cache_path.string(); auto status = NvExecutionProvider::RefitEngine(onnx_model_filename, - onnx_model_folder_path_, - weight_stripped_engine_cache, - make_secure_path_checks, - onnx_model_bytestream_, - onnx_model_bytestream_size_, - (*trt_engine_).get(), - true /* serialize refitted engine to disk */, - detailed_build_log_); + onnx_model_folder_path_, + weight_stripped_engine_cache, + make_secure_path_checks, + onnx_model_bytestream_, + onnx_model_bytestream_size_, + (*trt_engine_).get(), + true /* serialize refitted engine to disk */, + detailed_build_log_); if (status != Status::OK()) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, status.ErrorMessage()); } diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index dfcc343204295..b84d0a46d9ea6 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -554,12 +554,12 @@ ORT_API_STATUS_IMPL(SessionOptionsSetLoadCancellationFlag, _Inout_ OrtSessionOpt ORT_API(const OrtCompileApi*, GetCompileApi); ORT_API_STATUS_IMPL(SessionOptionsAppendExecutionProvider_Nv_TensorRT_RTX, - _In_ OrtSessionOptions* options, _In_ const OrtNvTensorRtRtxProviderOptions* nv_options); + _In_ OrtSessionOptions* options, _In_ const OrtNvTensorRtRtxProviderOptions* nv_options); ORT_API_STATUS_IMPL(CreateNvTensorRtRtxProviderOptions, _Outptr_ OrtNvTensorRtRtxProviderOptions** out); ORT_API_STATUS_IMPL(UpdateNvTensorRtRtxProviderOptions, _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - size_t num_keys); + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + size_t num_keys); ORT_API_STATUS_IMPL(GetNvTensorRtRtxProviderOptionsAsString, _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, _Inout_ OrtAllocator* allocator, _Outptr_ char** ptr); ORT_API(void, ReleaseNvTensorRtRtxProviderOptions, _Frees_ptr_opt_ OrtNvTensorRtRtxProviderOptions*); ORT_API_STATUS_IMPL(UpdateNvTensorRtRtxProviderOptionsWithValue, _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, _In_ const char* key, _In_ void* value); diff --git a/onnxruntime/core/session/provider_bridge_ort.cc b/onnxruntime/core/session/provider_bridge_ort.cc index 68ffd76257eea..5fb69ee8e2f2c 100644 --- a/onnxruntime/core/session/provider_bridge_ort.cc +++ b/onnxruntime/core/session/provider_bridge_ort.cc @@ -121,7 +121,7 @@ using EtwRegistrationManager_EtwInternalCallback = EtwRegistrationManager::EtwIn #include "core/providers/nv_tensorrt_rtx/nv_provider_factory.h" #include "core/providers/nv_tensorrt_rtx/nv_provider_options.h" -#if !defined(ORT_MINIMAL_BUILD) && ( defined(USE_TENSORRT) || defined(USE_NV) ) +#if !defined(ORT_MINIMAL_BUILD) && (defined(USE_TENSORRT) || defined(USE_NV)) #include "core/session/onnxruntime_session_options_config_keys.h" #endif @@ -2677,7 +2677,6 @@ ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensor #endif } - ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_Nv_TensorRT_RTX, _In_ OrtSessionOptions* options, _In_ const OrtNvTensorRtRtxProviderOptions* nv_options) { API_IMPL_BEGIN @@ -2692,7 +2691,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_Nv_TensorRT_R options->provider_factories.push_back(factory); // std::string extra_plugin_lib_paths = (nv_options == nullptr || nv_options->nv_extra_plugin_lib_paths == nullptr) ? "" : nv_options->nv_extra_plugin_lib_paths; - //AddTensorRTCustomOpDomainToSessionOption(options, extra_plugin_lib_paths); + // AddTensorRTCustomOpDomainToSessionOption(options, extra_plugin_lib_paths); return nullptr; API_IMPL_END @@ -2729,7 +2728,7 @@ ORT_API_STATUS_IMPL(OrtApis::UpdateNvTensorRtRtxProviderOptions, } onnxruntime::UpdateProviderInfo_Nv(nv_options, - reinterpret_cast(provider_options_map)); + reinterpret_cast(provider_options_map)); return nullptr; #else ORT_UNUSED_PARAMETER(nv_options); @@ -2770,10 +2769,10 @@ ORT_API_STATUS_IMPL(OrtApis::UpdateNvTensorRtRtxProviderOptionsWithValue, nv_options->user_compute_stream = value; } else if (strcmp(key, onnxruntime::nv::provider_option_names::kONNXBytestream) == 0) { nv_options->onnx_bytestream = value; - } else if (strcmp(key,onnxruntime::nv::provider_option_names::kONNXBytestreamSize) == 0) { + } else if (strcmp(key, onnxruntime::nv::provider_option_names::kONNXBytestreamSize) == 0) { nv_options->onnx_bytestream_size = *reinterpret_cast(value); } - return nullptr; + return nullptr; #else ORT_UNUSED_PARAMETER(nv_options); ORT_UNUSED_PARAMETER(key); @@ -2798,8 +2797,8 @@ ORT_API_STATUS_IMPL(OrtApis::GetNvTensorRtRtxProviderOptionsByName, ORT_API(void, OrtApis::ReleaseNvTensorRtRtxProviderOptions, _Frees_ptr_opt_ OrtNvTensorRtRtxProviderOptions* ptr) { #ifdef USE_NV -onnxruntime::ProviderInfo_Nv& provider_info = onnxruntime::GetProviderInfo_Nv(); -provider_info.ReleaseProviderOptions(ptr); + onnxruntime::ProviderInfo_Nv& provider_info = onnxruntime::GetProviderInfo_Nv(); + provider_info.ReleaseProviderOptions(ptr); #else ORT_UNUSED_PARAMETER(ptr); diff --git a/onnxruntime/core/session/provider_registration.cc b/onnxruntime/core/session/provider_registration.cc index f745a081809c0..9a6e626ff27d9 100644 --- a/onnxruntime/core/session/provider_registration.cc +++ b/onnxruntime/core/session/provider_registration.cc @@ -27,6 +27,9 @@ #include "core/providers/dml/dml_provider_factory_creator.h" #endif +#if defined(USE_NV) +#include "core/providers/nv_tensorrt_rtx/nv_provider_options.h" +#endif using namespace onnxruntime; namespace onnxruntime { @@ -98,6 +101,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, JS, VitisAI, CoreML, + NvTensorRtRtx, // TensorRt EP for RTX GPUs. }; struct EpToAppend { @@ -106,7 +110,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, const char* canonical_name = nullptr; }; - static std::array supported_eps = { + static std::array supported_eps = { EpToAppend{EpID::DML, "DML", kDmlExecutionProvider}, EpToAppend{EpID::QNN, "QNN", kQnnExecutionProvider}, EpToAppend{EpID::OpenVINO, "OpenVINO", kOpenVINOExecutionProvider}, @@ -118,7 +122,7 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, EpToAppend{EpID::JS, "JS", kJsExecutionProvider}, EpToAppend{EpID::VitisAI, "VitisAI", kVitisAIExecutionProvider}, EpToAppend{EpID::CoreML, "CoreML", kCoreMLExecutionProvider}, - }; + EpToAppend{EpID::NvTensorRtRtx, "NvTensorRtRtx", kNvTensorRTRTXExecutionProvider}}; ProviderOptions provider_options; OrtStatus* status = ParseProviderOptions(provider_options_keys, @@ -293,6 +297,21 @@ ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider, case EpID::CoreML: { #if defined(USE_COREML) options->provider_factories.push_back(CoreMLProviderFactoryCreator::Create(provider_options)); +#else + status = create_not_supported_status(); +#endif + break; + } + case EpID::NvTensorRtRtx: { +#if defined(USE_NV) + OrtNvTensorRtRtxProviderOptions* pNvOptions; + OrtApis::CreateNvTensorRtRtxProviderOptions(&pNvOptions); + OrtApis::UpdateNvTensorRtRtxProviderOptions(pNvOptions, + provider_options_keys, + provider_options_values, + num_keys); + OrtApis::SessionOptionsAppendExecutionProvider_Nv_TensorRT_RTX(options, pNvOptions); + #else status = create_not_supported_status(); #endif @@ -521,64 +540,62 @@ ORT_API(void, OrtApis::ReleaseTensorRTProviderOptions, _Frees_ptr_opt_ OrtTensor ORT_UNUSED_PARAMETER(ptr); } - - ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_Nv_TensorRT_RTX, - _In_ OrtSessionOptions* options, _In_ const OrtNvTensorRtRtxProviderOptions* nv_options) { -ORT_UNUSED_PARAMETER(options); -ORT_UNUSED_PARAMETER(nv_options); -return CreateNotEnabledStatus("Nv"); + _In_ OrtSessionOptions* options, _In_ const OrtNvTensorRtRtxProviderOptions* nv_options) { + ORT_UNUSED_PARAMETER(options); + ORT_UNUSED_PARAMETER(nv_options); + return CreateNotEnabledStatus("Nv"); } ORT_API_STATUS_IMPL(OrtApis::CreateNvTensorRtRtxProviderOptions, _Outptr_ OrtNvTensorRtRtxProviderOptions** out) { -ORT_UNUSED_PARAMETER(out); -return CreateNotEnabledStatus("Nv"); + ORT_UNUSED_PARAMETER(out); + return CreateNotEnabledStatus("Nv"); } ORT_API_STATUS_IMPL(OrtApis::UpdateNvTensorRtRtxProviderOptions, - _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, - _In_reads_(num_keys) const char* const* provider_options_keys, - _In_reads_(num_keys) const char* const* provider_options_values, - size_t num_keys) { -ORT_UNUSED_PARAMETER(nv_options); -ORT_UNUSED_PARAMETER(provider_options_keys); -ORT_UNUSED_PARAMETER(provider_options_values); -ORT_UNUSED_PARAMETER(num_keys); -return CreateNotEnabledStatus("Nv"); + _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, + _In_reads_(num_keys) const char* const* provider_options_keys, + _In_reads_(num_keys) const char* const* provider_options_values, + size_t num_keys) { + ORT_UNUSED_PARAMETER(nv_options); + ORT_UNUSED_PARAMETER(provider_options_keys); + ORT_UNUSED_PARAMETER(provider_options_values); + ORT_UNUSED_PARAMETER(num_keys); + return CreateNotEnabledStatus("Nv"); } ORT_API_STATUS_IMPL(OrtApis::GetNvTensorRtRtxProviderOptionsAsString, - _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, - _Inout_ OrtAllocator* allocator, - _Outptr_ char** ptr) { -ORT_UNUSED_PARAMETER(nv_options); -ORT_UNUSED_PARAMETER(allocator); -ORT_UNUSED_PARAMETER(ptr); -return CreateNotEnabledStatus("Nv"); + _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, + _Inout_ OrtAllocator* allocator, + _Outptr_ char** ptr) { + ORT_UNUSED_PARAMETER(nv_options); + ORT_UNUSED_PARAMETER(allocator); + ORT_UNUSED_PARAMETER(ptr); + return CreateNotEnabledStatus("Nv"); } ORT_API_STATUS_IMPL(OrtApis::UpdateNvTensorRtRtxProviderOptionsWithValue, - _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, - _In_ const char* key, - _In_ void* value) { -ORT_UNUSED_PARAMETER(nv_options); -ORT_UNUSED_PARAMETER(key); -ORT_UNUSED_PARAMETER(value); -return CreateNotEnabledStatus("Nv"); + _Inout_ OrtNvTensorRtRtxProviderOptions* nv_options, + _In_ const char* key, + _In_ void* value) { + ORT_UNUSED_PARAMETER(nv_options); + ORT_UNUSED_PARAMETER(key); + ORT_UNUSED_PARAMETER(value); + return CreateNotEnabledStatus("Nv"); } ORT_API_STATUS_IMPL(OrtApis::GetNvTensorRtRtxProviderOptionsByName, - _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, - _In_ const char* key, - _Outptr_ void** ptr) { -ORT_UNUSED_PARAMETER(nv_options); -ORT_UNUSED_PARAMETER(key); -ORT_UNUSED_PARAMETER(ptr); -return CreateNotEnabledStatus("Nv"); + _In_ const OrtNvTensorRtRtxProviderOptions* nv_options, + _In_ const char* key, + _Outptr_ void** ptr) { + ORT_UNUSED_PARAMETER(nv_options); + ORT_UNUSED_PARAMETER(key); + ORT_UNUSED_PARAMETER(ptr); + return CreateNotEnabledStatus("Nv"); } ORT_API(void, OrtApis::ReleaseNvTensorRtRtxProviderOptions, _Frees_ptr_opt_ OrtNvTensorRtRtxProviderOptions* ptr) { -ORT_UNUSED_PARAMETER(ptr); + ORT_UNUSED_PARAMETER(ptr); } ORT_API_STATUS_IMPL(OrtApis::SessionOptionsAppendExecutionProvider_MIGraphX, diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index 449ceed761b32..103da5f534ea7 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -40,7 +40,7 @@ namespace perftest { "\t-I: Generate tensor input binding. Free dimensions are treated as 1 unless overridden using -f.\n" "\t-c [parallel runs]: Specifies the (max) number of runs to invoke simultaneously. Default:1.\n" "\t-e [cpu|cuda|dnnl|tensorrt|openvino|dml|acl|nnapi|coreml|qnn|snpe|rocm|migraphx|xnnpack|vitisai|webgpu]: Specifies the provider 'cpu','cuda','dnnl','tensorrt', " - "'openvino', 'dml', 'acl', 'nnapi', 'coreml', 'qnn', 'snpe', 'rocm', 'migraphx', 'xnnpack', 'vitisai' or 'webgpu'. " + "'nvtensorrtrtx', 'openvino', 'dml', 'acl', 'nnapi', 'coreml', 'qnn', 'snpe', 'rocm', 'migraphx', 'xnnpack', 'vitisai' or 'webgpu'. " "Default:'cpu'.\n" "\t-b [tf|ort]: backend to use. Default:ort\n" "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n" @@ -264,7 +264,7 @@ static bool ParseDimensionOverride(std::basic_string& dim_identifier, test_config.machine_config.provider_type_name = onnxruntime::kVitisAIExecutionProvider; } else if (!CompareCString(optarg, ORT_TSTR("webgpu"))) { test_config.machine_config.provider_type_name = onnxruntime::kWebGpuExecutionProvider; - } else if (!CompareCString(optarg, ORT_TSTR("nv"))) { + } else if (!CompareCString(optarg, ORT_TSTR("nvtensorrtrtx"))) { test_config.machine_config.provider_type_name = onnxruntime::kNvTensorRTRTXExecutionProvider; } else { return false; diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index 06996fe2bf588..1eecb09da790d 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -213,7 +213,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device option_values.push_back(provider_option.second.c_str()); } Ort::Status status(api.UpdateNvTensorRtRtxProviderOptions(nv_options, - option_keys.data(), option_values.data(), option_keys.size())); + option_keys.data(), option_values.data(), option_keys.size())); if (!status.IsOK()) { OrtAllocator* allocator; char* options; diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 96b151b06bd7b..0a6af27da8bc2 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -2180,7 +2180,11 @@ def main(): # shared lib being build in a separate process. So we skip the testing if none of the primary EPs are built with ONNXRuntime # shared lib if args.enable_generic_interface and not ( - args.use_nv_tensorrt_rtx or args.use_tensorrt or args.use_openvino or args.use_vitisai or (args.use_qnn and args.use_qnn != "static_lib") + args.use_nv_tensorrt_rtx + or args.use_tensorrt + or args.use_openvino + or args.use_vitisai + or (args.use_qnn and args.use_qnn != "static_lib") ): args.test = False @@ -2291,8 +2295,6 @@ def main(): if args.use_tensorrt or args.use_nv_tensorrt_rtx: tensorrt_home = setup_tensorrt_vars(args) - - # if using migraphx, setup migraphx paths migraphx_home = setup_migraphx_vars(args) diff --git a/tools/ci_build/build_args.py b/tools/ci_build/build_args.py index 0f4b80244e55b..215ad77335083 100644 --- a/tools/ci_build/build_args.py +++ b/tools/ci_build/build_args.py @@ -665,7 +665,6 @@ def add_execution_provider_args(parser: argparse.ArgumentParser) -> None: nv_group = parser.add_argument_group("Nv Execution Provider") nv_group.add_argument("--use_nv_tensorrt_rtx", action="store_true", help="Enable Nv EP.") - # --- DirectML --- dml_group = parser.add_argument_group("DirectML Execution Provider (Windows)") dml_group.add_argument("--use_dml", action="store_true", help="Enable DirectML EP (Windows).")