diff --git a/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc b/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc index c9c89d298..5273f5b8a 100644 --- a/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc +++ b/sdk_v2/cpp/src/ep_detection/cuda_ep_bootstrapper.cc @@ -5,6 +5,7 @@ #include "ep_detection/ep_utils.h" #include "logger.h" #include "util/file_lock.h" +#include "utils.h" #include "http/http_download.h" #include "util/zip_extract.h" @@ -12,15 +13,11 @@ #include #include +#include #include #include #include -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif - namespace { constexpr const char* kPackageFileName = "cuda-ep.zip"; @@ -44,6 +41,7 @@ constexpr ExpectedBinary kExpectedBinaries[] = { constexpr const char* kRegistrationName = "Foundry.CUDA"; constexpr const char* kCudaProviderDll = "onnxruntime_providers_cuda.dll"; +constexpr const char* kCudaProviderOverrideEnv = "FOUNDRY_LOCAL_CUDA_EP_LIBRARY"; } // anonymous namespace @@ -82,6 +80,44 @@ bool CudaEpBootstrapper::DownloadAndRegister(bool force, auto zip_path = ep_dir.parent_path() / kPackageFileName; try { + auto override_path = Utils::GetEnv(kCudaProviderOverrideEnv); + if (override_path.has_value() && !override_path->empty()) { + std::filesystem::path provider_path(*override_path); + + if (!std::filesystem::exists(provider_path)) { + logger.Log(LogLevel::Warning, + fmt::format("CUDA EP: {} set but file does not exist ({})", + kCudaProviderOverrideEnv, provider_path.string())); + return false; + } + + if (progress_cb) { + progress_cb(name_, 90.0f); + } + + // Prepend the override directory to PATH so sibling dependency DLLs are discoverable, + // matching the normal install path. The provider DLL delay-loads CUDA/cuDNN dependencies. + PrependDirToProcessPath(provider_path.parent_path()); + + if (!register_ep_(kRegistrationName, provider_path)) { + logger.Log(LogLevel::Warning, + fmt::format("CUDA EP: ORT registration failed for override {}={}", + kCudaProviderOverrideEnv, provider_path.string())); + return false; + } + + registered_ = true; + + if (progress_cb) { + progress_cb(name_, 100.0f); + } + + logger.Log(LogLevel::Information, + fmt::format("CUDA EP: ready (override_env={} install_path={})", + kCudaProviderOverrideEnv, provider_path.string())); + return true; + } + // Cross-process lock to prevent concurrent installs FileLock lock(lock_path); @@ -153,18 +189,7 @@ bool CudaEpBootstrapper::DownloadAndRegister(bool force, // - onnxruntime_providers_cuda.dll delay-loads some dependencies // - onnxruntime-genai-cuda.dll is loaded later at model-load time // - ORT creates CUDA sessions after registration - { - DWORD len = GetEnvironmentVariableW(L"PATH", nullptr, 0); - std::wstring prev_path; - if (len > 0) { - prev_path.resize(len); - GetEnvironmentVariableW(L"PATH", prev_path.data(), len); - prev_path.resize(len - 1); // remove trailing null - } - - std::wstring new_path = ep_dir.wstring() + L";" + prev_path; - SetEnvironmentVariableW(L"PATH", new_path.c_str()); - } + PrependDirToProcessPath(ep_dir); #endif auto cuda_dll_path = ep_dir / kCudaProviderDll; diff --git a/sdk_v2/cpp/src/ep_detection/ep_utils.cc b/sdk_v2/cpp/src/ep_detection/ep_utils.cc index 7fa6524f4..7c49f6035 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_utils.cc +++ b/sdk_v2/cpp/src/ep_detection/ep_utils.cc @@ -11,6 +11,11 @@ #include #include +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#endif + namespace fl { bool VerifyEpPackage( @@ -40,4 +45,19 @@ bool VerifyEpPackage( return true; } +void PrependDirToProcessPath([[maybe_unused]] const std::filesystem::path& dir) { +#ifdef _WIN32 + DWORD len = GetEnvironmentVariableW(L"PATH", nullptr, 0); + std::wstring prev_path; + if (len > 0) { + prev_path.resize(len); + GetEnvironmentVariableW(L"PATH", prev_path.data(), len); + prev_path.resize(len - 1); // remove trailing null + } + + std::wstring new_path = dir.wstring() + L";" + prev_path; + SetEnvironmentVariableW(L"PATH", new_path.c_str()); +#endif +} + } // namespace fl diff --git a/sdk_v2/cpp/src/ep_detection/ep_utils.h b/sdk_v2/cpp/src/ep_detection/ep_utils.h index 634bb517e..c38027868 100644 --- a/sdk_v2/cpp/src/ep_detection/ep_utils.h +++ b/sdk_v2/cpp/src/ep_detection/ep_utils.h @@ -24,4 +24,14 @@ bool VerifyEpPackage( std::string_view ep_name, ILogger& logger); +/// Prepend @p dir to the process `PATH` environment variable for the lifetime of the process. +/// +/// EP provider libraries (CUDA, WebGPU) delay-load sibling dependency DLLs from their own directory, +/// and `RegisterExecutionProviderLibrary` loads the provider DLL eagerly. The directory must be on +/// `PATH` before registration so those dependencies are discoverable. This is a no-op on non-Windows +/// platforms. +/// +/// @param dir Directory to prepend to `PATH`. +void PrependDirToProcessPath(const std::filesystem::path& dir); + } // namespace fl diff --git a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc index 9aafb8382..eb4b3bedf 100644 --- a/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc +++ b/sdk_v2/cpp/src/ep_detection/webgpu_ep_bootstrapper.cc @@ -8,6 +8,7 @@ #include "logger.h" #include "util/file_lock.h" #include "util/sha256.h" +#include "utils.h" #include "util/zip_extract.h" #include @@ -16,14 +17,10 @@ #include #include #include +#include #include #include -#ifdef _WIN32 -#define WIN32_LEAN_AND_MEAN -#include -#endif - namespace { constexpr const char* kPackageFileName = "webgpu-ep.zip"; @@ -57,6 +54,7 @@ constexpr const char* kWebGpuProviderLib = "libonnxruntime_providers_webgpu.so"; #endif constexpr const char* kRegistrationName = "Foundry.WebGPU"; +constexpr const char* kWebGpuProviderOverrideEnv = "FOUNDRY_LOCAL_WEBGPU_EP_LIBRARY"; /// Parsed manifest entry for a single platform. struct ManifestPackageInfo { @@ -139,6 +137,44 @@ bool WebGpuEpBootstrapper::DownloadAndRegister(bool force, auto parent_dir = ep_dir.parent_path(); try { + auto override_path = Utils::GetEnv(kWebGpuProviderOverrideEnv); + if (override_path.has_value() && !override_path->empty()) { + std::filesystem::path provider_path(*override_path); + + if (!std::filesystem::exists(provider_path)) { + logger.Log(LogLevel::Warning, + fmt::format("WebGPU EP: {} set but file does not exist ({})", + kWebGpuProviderOverrideEnv, provider_path.string())); + return false; + } + + if (progress_cb) { + progress_cb(name_, 90.0f); + } + + // Prepend the override directory to PATH so sibling dependency DLLs are discoverable, + // matching the normal install path. The WebGPU EP may delay-load dependencies. + PrependDirToProcessPath(provider_path.parent_path()); + + if (!register_ep_(kRegistrationName, provider_path)) { + logger.Log(LogLevel::Warning, + fmt::format("WebGPU EP: ORT registration failed for override {}={}", + kWebGpuProviderOverrideEnv, provider_path.string())); + return false; + } + + registered_ = true; + + if (progress_cb) { + progress_cb(name_, 100.0f); + } + + logger.Log(LogLevel::Information, + fmt::format("WebGPU EP: ready (override_env={} install_path={})", + kWebGpuProviderOverrideEnv, provider_path.string())); + return true; + } + // Fetch manifest before acquiring lock (avoid holding lock during network I/O) auto manifest = FetchManifest(logger); @@ -232,18 +268,7 @@ bool WebGpuEpBootstrapper::DownloadAndRegister(bool force, #ifdef _WIN32 // Prepend the EP directory to PATH for the process lifetime. // WebGPU EP may delay-load additional dependencies from the same directory. - { - DWORD len = GetEnvironmentVariableW(L"PATH", nullptr, 0); - std::wstring prev_path; - if (len > 0) { - prev_path.resize(len); - GetEnvironmentVariableW(L"PATH", prev_path.data(), len); - prev_path.resize(len - 1); // remove trailing null - } - - std::wstring new_path = ep_dir.wstring() + L";" + prev_path; - SetEnvironmentVariableW(L"PATH", new_path.c_str()); - } + PrependDirToProcessPath(ep_dir); #endif auto provider_path = ep_dir / kWebGpuProviderLib; diff --git a/sdk_v2/cpp/src/manager.cc b/sdk_v2/cpp/src/manager.cc index 7d45dc71d..d0a4a15bb 100644 --- a/sdk_v2/cpp/src/manager.cc +++ b/sdk_v2/cpp/src/manager.cc @@ -41,6 +41,7 @@ namespace fl { namespace { +std::atomic s_ort_logger{nullptr}; std::atomic s_oga_logger{nullptr}; bool IsTruthyConfigValue(const std::string& value) { @@ -85,13 +86,13 @@ LogLevel MapOrtLogLevel(OrtLoggingLevel severity) { } } -void ORT_API_CALL OrtLogCallback(void* logger_param, +void ORT_API_CALL OrtLogCallback(void* /*logger_param*/, OrtLoggingLevel severity, const char* category, const char* logid, const char* code_location, const char* message) { - auto* logger = static_cast(logger_param); + auto* logger = s_ort_logger.load(std::memory_order_acquire); if (logger == nullptr) { return; } @@ -181,6 +182,7 @@ Manager::Manager(const Configuration& config) const auto logger_level = genai_verbose_logging ? LogLevel::Verbose : config_.log_level; logger_ = std::make_unique(logger_level, config_.logs_dir.value_or("")); + s_ort_logger.store(logger_.get(), std::memory_order_release); if (genai_verbose_logging) { SetOgaLogCallback(logger_.get()); @@ -203,7 +205,7 @@ Manager::Manager(const Configuration& config) { OrtStatus* status = ort_api_->CreateEnvWithCustomLogger( - OrtLogCallback, logger_.get(), GetDefaultOrtLoggingLevel(genai_verbose_logging), "foundry_local", &ort_env_); + OrtLogCallback, nullptr, GetDefaultOrtLoggingLevel(genai_verbose_logging), "foundry_local", &ort_env_); if (status != nullptr) { const char* msg = ort_api_->GetErrorMessage(status); std::string err = std::string("Failed to create OrtEnv: ") + (msg ? msg : "unknown"); @@ -355,6 +357,11 @@ Manager::~Manager() { } logger_->Log(LogLevel::Information, "Manager is being disposed."); + + // ORT may still emit late teardown logs from internal static cleanup after Manager destruction + // due to GenAI keeping the OrtEnv alive until the process exits. + // Clear s_ort_logger so OrtLogCallback does not dereference a dangling pointer and ignores late logs. + s_ort_logger.store(nullptr, std::memory_order_release); } Manager& Manager::Create(const Configuration& config) {