From fc2e20319b954fbc5044ff3696182a46b2678e0f Mon Sep 17 00:00:00 2001 From: keshavv27 Date: Tue, 16 Dec 2025 22:51:58 +0530 Subject: [PATCH 1/4] Fix TRT RTX plugin load path --- .../nv_execution_provider_custom_ops.cc | 44 ++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc index 5fe37a6c30e33..84c7062dae73d 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc @@ -23,6 +23,48 @@ namespace onnxruntime { extern TensorrtLogger& GetTensorrtLogger(bool verbose); +/// @brief Gets the directory path of the EP library that contains the current function +/// @return The directory path containing the EP library, or an empty string if the path cannot be determined. +static PathString GetEPLibraryDirectory() { +#ifdef _WIN32 + HMODULE hModule = NULL; + // Get handle to the DLL executing this code + if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(&GetEPLibraryDirectory), + &hModule)) { + return PathString(L""); + } + + wchar_t buffer[MAX_PATH]; + DWORD len = GetModuleFileNameW(hModule, buffer, MAX_PATH); + if (len == 0 || len >= MAX_PATH) { + return PathString(L""); + } + + std::wstring path(buffer); + size_t lastSlash = path.find_last_of(L"\\/"); + if (lastSlash != std::wstring::npos) { + return PathString(path.substr(0, lastSlash + 1)); + } + return PathString(path); +#else + // Linux and other Unix-like platforms + Dl_info dl_info; + + if (dladdr((void*)&GetEPLibraryDirectory, &dl_info) == 0 || dl_info.dli_fname == nullptr) { + return PathString(""); + } + + std::string so_path(dl_info.dli_fname); + size_t last_slash = so_path.find_last_of('/'); + if (last_slash != std::string::npos) { + return PathString(so_path.substr(0, last_slash + 1)); + } + return PathString(so_path); +#endif +} + /* * Create custom op domain list for TRT plugins. * @@ -76,7 +118,7 @@ common::Status CreateTensorRTCustomOpDomainList(std::vector& // This library contains GroupQueryAttention and RotaryEmbedding plugins for transformer models try { const auto& env = onnxruntime::GetDefaultEnv(); - auto external_plugin_path = env.GetRuntimePath() + + auto external_plugin_path = GetEPLibraryDirectory() + PathString(LIBRARY_PREFIX ORT_TSTR("tensorrt_plugins") LIBRARY_EXTENSION); void* external_plugin_handle = nullptr; auto status = env.LoadDynamicLibrary(external_plugin_path, false, &external_plugin_handle); From ab26c6fea0604bdab25f5f829056c38a9dffef4b Mon Sep 17 00:00:00 2001 From: keshavv27 Date: Tue, 16 Dec 2025 23:11:14 +0530 Subject: [PATCH 2/4] Add header files, fix error output --- .../nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc index 84c7062dae73d..c4b6d20e118ad 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc @@ -8,6 +8,12 @@ #include "nv_execution_provider_custom_ops.h" #include "nv_execution_provider.h" +#ifdef _WIN32 +#include +#else +#include +#endif + // The filename extension for a shared library is different per platform #ifdef _WIN32 #define LIBRARY_PREFIX @@ -125,7 +131,7 @@ common::Status CreateTensorRTCustomOpDomainList(std::vector& if (status.IsOK()) { LOGS_DEFAULT(INFO) << "[NvTensorRTRTX EP] External plugins loaded: tensorrt_plugins (GQA + RotaryEmbedding)"; } else { - LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] tensorrt_plugins library not found in runtime path (optional)"; + LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] tensorrt_plugins library not found in EP library path (optional)"; } } catch (const std::exception& e) { LOGS_DEFAULT(VERBOSE) << "[NvTensorRTRTX EP] tensorrt_plugins library not available: " << e.what(); From fe607b493b6e1cd7f4d18c96d26ca05600d31eeb Mon Sep 17 00:00:00 2001 From: keshavv27 Date: Wed, 17 Dec 2025 10:21:57 +0530 Subject: [PATCH 3/4] fix linting --- .../nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc index c4b6d20e118ad..acb450046b866 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc @@ -36,22 +36,22 @@ static PathString GetEPLibraryDirectory() { HMODULE hModule = NULL; // Get handle to the DLL executing this code if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, reinterpret_cast(&GetEPLibraryDirectory), &hModule)) { - return PathString(L""); + return PathString(L""); } wchar_t buffer[MAX_PATH]; DWORD len = GetModuleFileNameW(hModule, buffer, MAX_PATH); if (len == 0 || len >= MAX_PATH) { - return PathString(L""); + return PathString(L""); } std::wstring path(buffer); size_t lastSlash = path.find_last_of(L"\\/"); if (lastSlash != std::wstring::npos) { - return PathString(path.substr(0, lastSlash + 1)); + return PathString(path.substr(0, lastSlash + 1)); } return PathString(path); #else From aedf409ab593c04775205b8375beda54d299d4ea Mon Sep 17 00:00:00 2001 From: keshavv27 Date: Fri, 19 Dec 2025 15:28:57 +0530 Subject: [PATCH 4/4] use pathstring ctor, abstract fn and platform defs --- .../nv_execution_provider_custom_ops.cc | 61 +---------------- .../nv_tensorrt_rtx/nv_platform_utils.h | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+), 60 deletions(-) create mode 100644 onnxruntime/core/providers/nv_tensorrt_rtx/nv_platform_utils.h diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc index acb450046b866..90e488a1eda18 100644 --- a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_execution_provider_custom_ops.cc @@ -7,70 +7,11 @@ #include "core/framework/provider_options.h" #include "nv_execution_provider_custom_ops.h" #include "nv_execution_provider.h" - -#ifdef _WIN32 -#include -#else -#include -#endif - -// The filename extension for a shared library is different per platform -#ifdef _WIN32 -#define LIBRARY_PREFIX -#define LIBRARY_EXTENSION ORT_TSTR(".dll") -#elif defined(__APPLE__) -#define LIBRARY_PREFIX "lib" -#define LIBRARY_EXTENSION ".dylib" -#else -#define LIBRARY_PREFIX "lib" -#define LIBRARY_EXTENSION ".so" -#endif +#include "nv_platform_utils.h" namespace onnxruntime { extern TensorrtLogger& GetTensorrtLogger(bool verbose); -/// @brief Gets the directory path of the EP library that contains the current function -/// @return The directory path containing the EP library, or an empty string if the path cannot be determined. -static PathString GetEPLibraryDirectory() { -#ifdef _WIN32 - HMODULE hModule = NULL; - // Get handle to the DLL executing this code - if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | - GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, - reinterpret_cast(&GetEPLibraryDirectory), - &hModule)) { - return PathString(L""); - } - - wchar_t buffer[MAX_PATH]; - DWORD len = GetModuleFileNameW(hModule, buffer, MAX_PATH); - if (len == 0 || len >= MAX_PATH) { - return PathString(L""); - } - - std::wstring path(buffer); - size_t lastSlash = path.find_last_of(L"\\/"); - if (lastSlash != std::wstring::npos) { - return PathString(path.substr(0, lastSlash + 1)); - } - return PathString(path); -#else - // Linux and other Unix-like platforms - Dl_info dl_info; - - if (dladdr((void*)&GetEPLibraryDirectory, &dl_info) == 0 || dl_info.dli_fname == nullptr) { - return PathString(""); - } - - std::string so_path(dl_info.dli_fname); - size_t last_slash = so_path.find_last_of('/'); - if (last_slash != std::string::npos) { - return PathString(so_path.substr(0, last_slash + 1)); - } - return PathString(so_path); -#endif -} - /* * Create custom op domain list for TRT plugins. * diff --git a/onnxruntime/core/providers/nv_tensorrt_rtx/nv_platform_utils.h b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_platform_utils.h new file mode 100644 index 0000000000000..f3298a8449157 --- /dev/null +++ b/onnxruntime/core/providers/nv_tensorrt_rtx/nv_platform_utils.h @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include "core/common/path_string.h" + +#ifdef _WIN32 +#include +#else +#include +#endif + +// The filename extension for a shared library is different per platform +#ifdef _WIN32 +#define LIBRARY_PREFIX +#define LIBRARY_EXTENSION ORT_TSTR(".dll") +#elif defined(__APPLE__) +#define LIBRARY_PREFIX "lib" +#define LIBRARY_EXTENSION ".dylib" +#else +#define LIBRARY_PREFIX "lib" +#define LIBRARY_EXTENSION ".so" +#endif + +namespace onnxruntime { +inline PathString GetEPLibraryDirectory() { +#ifdef _WIN32 + HMODULE hModule = NULL; + // Get handle to the DLL executing this code + if (!GetModuleHandleExW(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | + GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(GetEPLibraryDirectory), + &hModule)) { + return PathString(); + } + + wchar_t buffer[MAX_PATH]; + DWORD len = GetModuleFileNameW(hModule, buffer, MAX_PATH); + if (len == 0 || len >= MAX_PATH) { + return PathString(); + } + + std::wstring path(buffer); + size_t lastSlash = path.find_last_of(L"\\/"); + if (lastSlash != std::wstring::npos) { + return PathString(path.substr(0, lastSlash + 1)); + } + return PathString(); +#else + // Linux and other Unix-like platforms + Dl_info dl_info; + + if (dladdr((void*)&GetEPLibraryDirectory, &dl_info) == 0 || dl_info.dli_fname == nullptr) { + return PathString(); + } + + std::string so_path(dl_info.dli_fname); + size_t last_slash = so_path.find_last_of('/'); + if (last_slash != std::string::npos) { + return PathString(so_path.substr(0, last_slash + 1)); + } + return PathString(); +#endif +} +} // namespace onnxruntime