From 671056d2edea5d2322963a2d222003cebad5a455 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Tue, 21 Jul 2026 19:09:01 +0000 Subject: [PATCH] support no cufft installation --- .github/workflows/linux_cuda_no_cudnn.yml | 52 ++++- .github/workflows/windows_cuda_no_cudnn.yml | 18 +- cmake/onnxruntime_providers_cuda.cmake | 4 +- cmake/onnxruntime_providers_cuda_plugin.cmake | 4 +- cmake/onnxruntime_unittests.cmake | 2 +- docs/CUDA_cuDNN_Optional_Design.md | 34 +++ docs/cuda_plugin_ep/cuda_plugin_ep_design.md | 8 +- onnxruntime/core/providers/cuda/cuda_call.cc | 13 ++ .../core/providers/cuda/cufft_loader.cc | 209 ++++++++++++++++++ .../core/providers/cuda/cufft_loader.h | 51 +++++ onnxruntime/core/providers/cuda/cufft_stub.cc | 54 +++++ .../cuda/plugin/cuda_kernel_adapter.h | 6 + 12 files changed, 446 insertions(+), 9 deletions(-) create mode 100644 onnxruntime/core/providers/cuda/cufft_loader.cc create mode 100644 onnxruntime/core/providers/cuda/cufft_loader.h create mode 100644 onnxruntime/core/providers/cuda/cufft_stub.cc diff --git a/.github/workflows/linux_cuda_no_cudnn.yml b/.github/workflows/linux_cuda_no_cudnn.yml index 5082095f2ee48..ba1e956d47690 100644 --- a/.github/workflows/linux_cuda_no_cudnn.yml +++ b/.github/workflows/linux_cuda_no_cudnn.yml @@ -97,14 +97,15 @@ jobs: done < perms.txt fi - - name: Verify CUDA provider has no direct cuDNN dependency + - name: Verify CUDA provider has no direct cuDNN or cuFFT dependency run: | docker run --rm --gpus all \ -v "${{ runner.temp }}/Release:/build/Release" \ "${{ steps.build_docker_image_step.outputs.full-image-name }}" \ bash -lc 'set -euo pipefail ldd /build/Release/Release/libonnxruntime_providers_cuda.so | tee /tmp/ldd.txt - ! grep -i cudnn /tmp/ldd.txt' + ! grep -i cudnn /tmp/ldd.txt + ! grep -i cufft /tmp/ldd.txt' - name: Run no-cuDNN CUDA EP op smoke test run: | @@ -137,6 +138,24 @@ jobs: exit 1 fi + # Likewise remove cuFFT so the smoke test proves the provider loads and runs + # non-FFT operators when cuFFT is physically unavailable (it is loaded lazily and + # only needed by the Rfft/Irfft contrib ops). + for root in /usr /opt /lib /lib64; do + if [ -e "$root" ]; then + find "$root" -name "libcufft*.so*" -print -delete 2>/dev/null || true + fi + done + ldconfig + if ldconfig -p | grep -qi cufft; then + echo "cuFFT remains available in the dynamic linker cache" >&2 + exit 1 + fi + if find /usr /opt /lib /lib64 -name "libcufft*.so*" -print -quit 2>/dev/null | grep -q .; then + echo "cuFFT runtime libraries remain in the no-cuDNN test container" >&2 + exit 1 + fi + WHEEL_PATH=$(find /build/Release/Release/dist -type f -name "onnxruntime_gpu-*.whl" | head -n 1) if [ -z "$WHEEL_PATH" ]; then echo "No built onnxruntime GPU wheel found under /build/Release/Release/dist" >&2 @@ -255,5 +274,32 @@ jobs: np.argmin(data, axis=1).reshape(2, 1).astype(np.int64), ) - print("CUDA no-cuDNN op smoke tests passed") + # Rfft requires cuFFT, which was physically removed above. With CPU fallback + # disabled the node must run on CUDA and fail cleanly with NOT_IMPLEMENTED + # (proving cuFFT is optional and its absence does not crash the provider). + rfft_model = helper.make_model( + helper.make_graph( + [helper.make_node("Rfft", ["x"], ["y"], domain="com.microsoft", signal_ndim=1, onesided=1)], + "no_cufft_rfft", + [helper.make_tensor_value_info("x", TensorProto.FLOAT, [2, 4])], + [helper.make_tensor_value_info("y", TensorProto.FLOAT, [2, 3, 2])], + ), + opset_imports=[helper.make_opsetid("", 17), helper.make_opsetid("com.microsoft", 1)], + ) + rfft_model.ir_version = 10 + rfft_options = ort.SessionOptions() + rfft_options.add_session_config_entry("session.disable_cpu_ep_fallback", "1") + try: + rfft_sess = ort.InferenceSession( + rfft_model.SerializeToString(), sess_options=rfft_options, providers=providers + ) + rfft_sess.run(None, {"x": np.random.rand(2, 4).astype(np.float32)}) + except Exception as exc: # noqa: BLE001 + message = str(exc).lower() + assert "cufft" in message, "Rfft without cuFFT raised an unexpected error: " + str(exc) + print("[no-cuFFT] Rfft correctly failed: " + str(exc).splitlines()[0]) + else: + raise AssertionError("Rfft unexpectedly succeeded while cuFFT was unavailable") + + print("CUDA no-cuDNN/no-cuFFT op smoke tests passed") PY' diff --git a/.github/workflows/windows_cuda_no_cudnn.yml b/.github/workflows/windows_cuda_no_cudnn.yml index 491651c3786a5..c8ebca6b5d561 100644 --- a/.github/workflows/windows_cuda_no_cudnn.yml +++ b/.github/workflows/windows_cuda_no_cudnn.yml @@ -214,10 +214,20 @@ jobs: dir shell: pwsh + - name: Remove cuFFT runtime DLLs + shell: pwsh + run: | + $cudaRoot = Join-Path $env:RUNNER_TEMP "v13.0" + Get-ChildItem -Path $cudaRoot -Recurse -Include "cufft*.dll" | Remove-Item -Force + if (Get-ChildItem -Path $cudaRoot -Recurse -Include "cufft*.dll" -ErrorAction SilentlyContinue) { + Write-Error "cuFFT runtime DLLs must not be present in the no-cuDNN test environment" + exit 1 + } + - name: Add CUDA to PATH shell: pwsh run: | - Write-Host "Adding CUDA to PATH without adding any cuDNN directory" + Write-Host "Adding CUDA to PATH without cuDNN or cuFFT runtime DLLs" Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\bin" Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\bin\x64" Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\extras\CUPTI\lib64" @@ -237,7 +247,7 @@ jobs: shell: pwsh run: nvidia-smi - - name: Verify CUDA plugin has no direct cuDNN dependency + - name: Verify CUDA plugin has no direct cuDNN or cuFFT dependency shell: pwsh run: | $pluginPath = "${{ runner.temp }}\build\Release\Release\onnxruntime_providers_cuda.dll" @@ -251,6 +261,10 @@ jobs: Write-Error "CUDA plugin EP has a direct cuDNN dependency" exit 1 } + if (Select-String -Path $env:RUNNER_TEMP\cuda_plugin_dependents.txt -Pattern "cufft" -SimpleMatch -Quiet) { + Write-Error "CUDA plugin EP has a direct cuFFT dependency" + exit 1 + } - name: Run CUDA Plugin EP Python Tests without cuDNN working-directory: ${{ github.workspace }}\onnxruntime\test\python\transformers diff --git a/cmake/onnxruntime_providers_cuda.cmake b/cmake/onnxruntime_providers_cuda.cmake index f9487fb2c8b5e..0b787d51447e8 100644 --- a/cmake/onnxruntime_providers_cuda.cmake +++ b/cmake/onnxruntime_providers_cuda.cmake @@ -402,7 +402,9 @@ endif() target_compile_definitions(${target} PRIVATE NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING) target_include_directories(${target} PRIVATE ${CUDNN_INCLUDE_DIR}) - target_link_libraries(${target} PRIVATE CUDA::cublasLt CUDA::cublas cudnn_frontend CUDA::curand CUDA::cufft CUDA::cudart CUDA::cuda_driver + # cuDNN and cuFFT are loaded dynamically at runtime (see cudnn_stub.cc / cufft_stub.cc), so they are + # intentionally not linked here to avoid a hard runtime dependency when the related ops are not used. + target_link_libraries(${target} PRIVATE CUDA::cublasLt CUDA::cublas cudnn_frontend CUDA::curand CUDA::cudart CUDA::cuda_driver ${ABSEIL_LIBS} ${ONNXRUNTIME_PROVIDERS_SHARED} Boost::mp11 safeint_interface) endif() diff --git a/cmake/onnxruntime_providers_cuda_plugin.cmake b/cmake/onnxruntime_providers_cuda_plugin.cmake index a2755f80c6027..93435a6754fb4 100644 --- a/cmake/onnxruntime_providers_cuda_plugin.cmake +++ b/cmake/onnxruntime_providers_cuda_plugin.cmake @@ -386,11 +386,13 @@ onnxruntime_add_include_to_target( add_dependencies(onnxruntime_providers_cuda_plugin ${onnxruntime_EXTERNAL_DEPENDENCIES}) # Link libraries +# cuDNN and cuFFT are loaded dynamically at runtime (see cudnn_stub.cc / cufft_stub.cc, which are +# picked up by the GLOB above), so they are intentionally not linked here to avoid a hard runtime +# dependency when the related ops are not used. target_link_libraries(onnxruntime_providers_cuda_plugin PRIVATE CUDA::cudart CUDA::cublas CUDA::cublasLt - CUDA::cufft CUDA::cuda_driver cudnn_frontend Boost::mp11 diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index d6bae69425ef5..81bb0878f5017 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -1051,6 +1051,7 @@ if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND onnxruntime_BUILD_CUDA_EP_AS_P "${ONNXRUNTIME_ROOT}/core/providers/cuda/cuda_utils.cu" "${ONNXRUNTIME_ROOT}/core/providers/cuda/cudnn_common.cc" "${ONNXRUNTIME_ROOT}/core/providers/cuda/cudnn_loader.cc" + "${ONNXRUNTIME_ROOT}/core/providers/cuda/cufft_loader.cc" "${ONNXRUNTIME_ROOT}/core/providers/cuda/reduction/reduction_functions.cc" "${ONNXRUNTIME_ROOT}/core/providers/cuda/reduction/reduction_functions.cu" "${TEST_SRC_DIR}/providers/cuda/test_cases/cuda_plugin_test_shims.cc" @@ -1408,7 +1409,6 @@ block() CUDA::cublas CUDA::cublasLt CUDA::curand - CUDA::cufft CUDA::cuda_driver CUDNN::cudnn cudnn_frontend) diff --git a/docs/CUDA_cuDNN_Optional_Design.md b/docs/CUDA_cuDNN_Optional_Design.md index 8d24b8db30186..1d761d31b72b0 100644 --- a/docs/CUDA_cuDNN_Optional_Design.md +++ b/docs/CUDA_cuDNN_Optional_Design.md @@ -358,6 +358,40 @@ TensorRT / NV‑RTX EPs are untouched and continue to link cuDNN as before. (If and the shimmed CUDA EP are in the same process, symbol collision must be avoided — see §8 Risks.) +### 3.6 Applying the same pattern to cuFFT + +The identical shim + lazy‑loader technique is used to drop the hard dependency on **cuFFT** +(`libcufft.so.*` / `cufft64_*.dll`). cuFFT is only needed by the FFT contrib operators +(`Rfft` / `Irfft` in `contrib_ops/cuda/math`), which use a small, enumerable set of entry +points: `cufftCreate`, `cufftDestroy`, `cufftSetStream`, `cufftXtMakePlanMany`, and +`cufftXtExec`. + +Implementation: + +- **Loader** — `onnxruntime/core/providers/cuda/cufft_loader.{h,cc}` defines a + `CufftLibrary` singleton that mirrors `CudnnLibrary`: it lazily `dlopen`/`LoadLibrary`s the + cuFFT runtime, resolves symbols on demand, caches them, and exposes `Available()` / `Error()`. + It uses the same security‑conscious search behavior as the cuDNN loader (Windows + `LOAD_LIBRARY_SEARCH_DEFAULT_DIRS` plus a PATH fallback that never loads from the current + working directory). The candidate library name is selected at **compile time** from the + `CUDA_VERSION` macro (cuFFT 12 for CUDA 13.x, cuFFT 11 for CUDA 12.x), because cuFFT's + SONAME/DLL version tracks the CUDA major version. Unsupported CUDA major versions fail at + compile time until their cuFFT library mapping is added explicitly. +- **Shim** — `onnxruntime/core/providers/cuda/cufft_stub.cc` defines the five entry points + above as trampolines that forward to the resolved symbol (returning `CUFFT_INTERNAL_ERROR` + when the library is unavailable), so ORT's direct calls link against *our* definitions and + the final binary has **no import entry for libcufft**. +- **Availability guard** — `CudaCall` (in `cuda_call.cc`) and the plugin's + `CUFFT_RETURN_IF_ERROR` macro check `CufftLibrary::Available()` and return a clear + `NOT_IMPLEMENTED` status ("cuFFT is unavailable …") when cuFFT is missing, exactly like the + cuDNN path. + +Unlike cuDNN, cuFFT needs **no provider option** (there is no `enable_cufft`): it is always +loaded lazily on first FFT‑op use, and there is no `cufft_frontend` equivalent to configure. +cuFFT headers (part of the CUDA toolkit) are still required at build time, but `CUDA::cufft` +is no longer linked in `onnxruntime_providers_cuda.cmake`, +`onnxruntime_providers_cuda_plugin.cmake`, or the CUDA unit‑test target. + --- ## 4. Phase 1 — Make cuDNN optional (no new kernels) diff --git a/docs/cuda_plugin_ep/cuda_plugin_ep_design.md b/docs/cuda_plugin_ep/cuda_plugin_ep_design.md index 7dfa2bd2d8667..a4c67342194e2 100644 --- a/docs/cuda_plugin_ep/cuda_plugin_ep_design.md +++ b/docs/cuda_plugin_ep/cuda_plugin_ep_design.md @@ -43,6 +43,12 @@ There is intentionally no provider option for a custom cuDNN DLL/SO path. Provid No-cuDNN plugin validation runs `test_cuda_plugin_ep.py` with `ORT_TEST_CUDA_PLUGIN_EP=1` and `ORT_TEST_CUDA_PLUGIN_NO_CUDNN=1`. That mode passes `enable_cudnn=0` to plugin sessions and skips tests for operators that still require cuDNN in the current implementation. Non-cuDNN operator coverage, plugin registration, device enumeration, graph assignment, CUDA graph, I/O binding, and profiling tests continue to run. +### 2.1.2 Optional cuFFT Runtime Dependency + +The CUDA Plugin EP applies the same optional-dependency model to **cuFFT**. cuFFT is only used by the FFT contrib operators (`Rfft` / `Irfft`), so the plugin shared library must not link directly to cuFFT or contain a cuFFT DLL/SO in its dynamic dependency table. cuFFT is loaded lazily through the ORT cuFFT loader (`CufftLibrary`, `core/providers/cuda/cufft_loader.{h,cc}`) on first FFT-op use; the entry points ORT calls are provided by `cufft_stub.cc` trampolines that forward to the resolved symbols. + +Unlike cuDNN, there is **no provider option** for cuFFT (no `enable_cufft`) and no frontend library to configure. When cuFFT is unavailable, `CUFFT_RETURN_IF_ERROR` (and `CudaCall`) return `NOT_IMPLEMENTED` with an actionable message, so FFT ops fail cleanly while all other operators keep working. cuFFT headers (part of the CUDA toolkit) are still required at build time, but `CUDA::cufft` is no longer linked. The loader picks the cuFFT library name at compile time from the `CUDA_VERSION` macro (cuFFT 12 for CUDA 13.x, cuFFT 11 for CUDA 12.x); unsupported CUDA major versions fail at compile time until their mapping is added. + ### 2.2 Preprocessor Defines Each build target uses different preprocessor defines that control how framework types are resolved: @@ -651,7 +657,7 @@ Use `onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=OFF` when you need to build the origina ### 9.3 Plugin Independence -The plugin build's `libonnxruntime_providers_cuda.so` is **fully self-contained**. It does not depend on `libonnxruntime_providers_shared.so` at load time. It statically links against `onnxruntime_framework`, `onnxruntime_graph`, `onnxruntime_common`, `onnxruntime_mlas`, `onnxruntime_flatbuffers`, and links dynamically against CUDA (`cudart`, `cublas`, `cublasLt`, `cufft`) and protobuf. cuDNN is loaded lazily only when enabled and available at runtime. Communication with the ORT runtime happens exclusively through the C API (`OrtApi`/`OrtEpApi`) passed at load time. +The plugin build's `libonnxruntime_providers_cuda.so` is **fully self-contained**. It does not depend on `libonnxruntime_providers_shared.so` at load time. It statically links against `onnxruntime_framework`, `onnxruntime_graph`, `onnxruntime_common`, `onnxruntime_mlas`, `onnxruntime_flatbuffers`, and links dynamically against CUDA (`cudart`, `cublas`, `cublasLt`) and protobuf. cuDNN and cuFFT are loaded lazily only when available at runtime (cuDNN additionally gated by `enable_cudnn`), so neither appears in the library's dynamic dependency table. Communication with the ORT runtime happens exclusively through the C API (`OrtApi`/`OrtEpApi`) passed at load time. ### 9.4 Build Outputs diff --git a/onnxruntime/core/providers/cuda/cuda_call.cc b/onnxruntime/core/providers/cuda/cuda_call.cc index d27dbf266d183..23143141f32c1 100644 --- a/onnxruntime/core/providers/cuda/cuda_call.cc +++ b/onnxruntime/core/providers/cuda/cuda_call.cc @@ -11,6 +11,7 @@ #endif #ifndef USE_CUDA_MINIMAL #include "core/providers/cuda/cudnn_loader.h" +#include "core/providers/cuda/cufft_loader.h" #endif #include @@ -107,6 +108,18 @@ std::conditional_t CudaCall( } } } + if constexpr (std::is_same_v) { + if (!cuda::CufftLibrary::Get().Available()) { + auto status = ORT_MAKE_STATUS(ONNXRUNTIME, NOT_IMPLEMENTED, + "cuFFT is unavailable for CUDA Execution Provider: ", + cuda::CufftLibrary::Get().Error()); + if constexpr (THRW) { + ORT_THROW(status.ErrorMessage()); + } else { + return status; + } + } + } #endif try { #ifdef _WIN32 diff --git a/onnxruntime/core/providers/cuda/cufft_loader.cc b/onnxruntime/core/providers/cuda/cufft_loader.cc new file mode 100644 index 0000000000000..6c9ed625e1e94 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cufft_loader.cc @@ -0,0 +1,209 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/providers/cuda/cufft_loader.h" + +#ifndef USE_CUDA_MINIMAL + +#include + +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif + +namespace { + +// cuFFT's SONAME/DLL version tracks the CUDA major version it ships with: +// CUDA 12.x -> cuFFT 11, CUDA 13.x -> cuFFT 12. Select the name that matches the +// CUDA toolkit this library was built against (CUDA_VERSION comes from cuda_pch.h). +#if CUDA_VERSION >= 13000 && CUDA_VERSION < 14000 +#ifdef _WIN32 +constexpr const char* kCufftLibraryName = "cufft64_12.dll"; +#else +constexpr const char* kCufftLibraryName = "libcufft.so.12"; +#endif +#elif CUDA_VERSION >= 12000 && CUDA_VERSION < 13000 +#ifdef _WIN32 +constexpr const char* kCufftLibraryName = "cufft64_11.dll"; +#else +constexpr const char* kCufftLibraryName = "libcufft.so.11"; +#endif +#else +#error Unsupported CUDA_VERSION for dynamic cuFFT loading +#endif + +#ifdef _WIN32 +// Search the directories listed in the PATH environment variable for the given +// library and, if found, load it by its full path. Loading by full path (rather +// than letting the loader search) preserves the historical PATH-based cuFFT +// discovery without ever loading from the current working directory, which the +// LOAD_LIBRARY_SEARCH_DEFAULT_DIRS-only search excludes for security reasons. +HMODULE LoadLibraryFromPathEnv(const std::string& candidate) { + DWORD length = GetEnvironmentVariableA("PATH", nullptr, 0); + if (length == 0) { + return nullptr; + } + + std::string path_value(length, '\0'); + length = GetEnvironmentVariableA("PATH", path_value.data(), length); + if (length == 0) { + return nullptr; + } + path_value.resize(length); + + size_t start = 0; + while (start <= path_value.size()) { + size_t end = path_value.find(';', start); + if (end == std::string::npos) { + end = path_value.size(); + } + + std::string dir = path_value.substr(start, end - start); + start = end + 1; + if (dir.empty()) { + continue; + } + + if (dir.back() != '\\' && dir.back() != '/') { + dir.push_back('\\'); + } + std::string full_path = dir + candidate; + + if (GetFileAttributesA(full_path.c_str()) == INVALID_FILE_ATTRIBUTES) { + continue; + } + + HMODULE handle = LoadLibraryExA(full_path.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + if (handle != nullptr) { + return handle; + } + } + + return nullptr; +} +#endif + +void* LoadLibraryCandidate(const std::string& candidate, std::string& error) { +#ifdef _WIN32 + // Use LOAD_LIBRARY_SEARCH_DEFAULT_DIRS so cuFFT is resolved only from the + // application directory, %WINDIR%\System32, and directories added via + // AddDllDirectory/SetDefaultDllDirectories. This deliberately excludes the + // current working directory from the search order to avoid loading an + // attacker-controlled DLL from the process CWD. + HMODULE handle = LoadLibraryExA(candidate.c_str(), nullptr, LOAD_LIBRARY_SEARCH_DEFAULT_DIRS); + if (handle == nullptr) { + // Fall back to searching the directories listed in PATH (loading by full + // path), matching the pre-existing OS-loader behavior when cuFFT was a + // direct import dependency. The current working directory is never searched. + handle = LoadLibraryFromPathEnv(candidate); + } + if (handle == nullptr) { + error = "LoadLibrary failed for " + candidate + " with error " + std::to_string(GetLastError()); + } + return reinterpret_cast(handle); +#else + dlerror(); + void* handle = dlopen(candidate.c_str(), RTLD_NOW | RTLD_LOCAL); + if (handle == nullptr) { + const char* dl_error = dlerror(); + error = "dlopen failed for " + candidate + ": " + (dl_error != nullptr ? dl_error : "unknown error"); + } + return handle; +#endif +} + +void* GetLibrarySymbol(void* handle, const char* symbol, std::string& error) { +#ifdef _WIN32 + void* address = reinterpret_cast(GetProcAddress(reinterpret_cast(handle), symbol)); + if (address == nullptr) { + error = "GetProcAddress failed for " + std::string(symbol) + " with error " + std::to_string(GetLastError()); + } + return address; +#else + dlerror(); + void* address = dlsym(handle, symbol); + const char* dl_error = dlerror(); + if (address == nullptr || dl_error != nullptr) { + error = "dlsym failed for " + std::string(symbol) + ": " + (dl_error != nullptr ? dl_error : "unknown error"); + } + return address; +#endif +} + +} // namespace + +namespace onnxruntime::cuda { + +CufftLibrary& CufftLibrary::Get() { + static CufftLibrary library; + return library; +} + +bool CufftLibrary::Available() { + return EnsureLoaded(); +} + +std::string CufftLibrary::Error() const { + std::lock_guard lock(mutex_); + return error_.empty() ? CufftUnavailableErrorString() : error_; +} + +bool CufftLibrary::EnsureLoaded() { + std::lock_guard lock(mutex_); + if (load_attempted_) { + return available_; + } + + load_attempted_ = true; + handle_ = LoadLibraryCandidate(kCufftLibraryName, error_); + if (handle_ != nullptr) { + available_ = true; + error_.clear(); + return true; + } + + available_ = false; + if (error_.empty()) { + error_ = "cuFFT library was not found"; + } + return false; +} + +void* CufftLibrary::ResolveSymbol(const char* symbol) { + { + std::lock_guard lock(mutex_); + auto it = symbols_.find(symbol); + if (it != symbols_.end()) { + return it->second; + } + } + + if (!EnsureLoaded()) { + return nullptr; + } + + std::lock_guard lock(mutex_); + std::string symbol_error; + void* address = GetLibrarySymbol(handle_, symbol, symbol_error); + if (address == nullptr) { + available_ = false; + error_ = symbol_error; + return nullptr; + } + + symbols_.emplace(symbol, address); + return address; +} + +const char* CufftUnavailableErrorString() { + return "cuFFT is not available. Install cuFFT or update the system library search path to enable FFT (Rfft/Irfft) operators."; +} + +} // namespace onnxruntime::cuda + +#endif // USE_CUDA_MINIMAL diff --git a/onnxruntime/core/providers/cuda/cufft_loader.h b/onnxruntime/core/providers/cuda/cufft_loader.h new file mode 100644 index 0000000000000..f7be4b4357470 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cufft_loader.h @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#ifndef USE_CUDA_MINIMAL + +#include +#include +#include + +#include "core/providers/cuda/cuda_pch.h" + +namespace onnxruntime::cuda { + +// Lazily loads the cuFFT runtime library and resolves its symbols on demand so +// that the CUDA Execution Provider does not carry a hard link-time dependency on +// cuFFT. cuFFT is only required by the FFT contrib ops (Rfft/Irfft); when the +// library is not present those ops fail with NOT_IMPLEMENTED while the rest of +// the provider keeps working. +class CufftLibrary { + public: + static CufftLibrary& Get(); + + bool Available(); + std::string Error() const; + + template + T Resolve(const char* symbol) { + return reinterpret_cast(ResolveSymbol(symbol)); + } + + private: + CufftLibrary() = default; + + bool EnsureLoaded(); + void* ResolveSymbol(const char* symbol); + + mutable std::mutex mutex_; + bool load_attempted_{false}; + bool available_{false}; + std::string error_; + void* handle_{nullptr}; + std::unordered_map symbols_; +}; + +const char* CufftUnavailableErrorString(); + +} // namespace onnxruntime::cuda + +#endif // USE_CUDA_MINIMAL diff --git a/onnxruntime/core/providers/cuda/cufft_stub.cc b/onnxruntime/core/providers/cuda/cufft_stub.cc new file mode 100644 index 0000000000000..a1cc195375556 --- /dev/null +++ b/onnxruntime/core/providers/cuda/cufft_stub.cc @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Provides definitions for the small set of cuFFT entry points used by the CUDA +// Execution Provider. Each stub forwards to the matching symbol resolved from the +// dynamically loaded cuFFT runtime library. Linking against these stubs (instead +// of the cuFFT import library) removes the hard runtime dependency on cuFFT so +// that models without FFT (Rfft/Irfft) ops can run without cuFFT installed. + +#include "core/providers/cuda/cufft_loader.h" + +#ifndef USE_CUDA_MINIMAL + +#include "cufft.h" +#include "cufftXt.h" + +#define ORT_CUFFT_FORWARD_RESULT(name, ...) \ + using Fn = decltype(&name); \ + auto fn = onnxruntime::cuda::CufftLibrary::Get().Resolve(#name); \ + return fn != nullptr ? fn(__VA_ARGS__) : CUFFT_INTERNAL_ERROR + +extern "C" { + +cufftResult CUFFTAPI cufftCreate(cufftHandle* handle) { + ORT_CUFFT_FORWARD_RESULT(cufftCreate, handle); +} + +cufftResult CUFFTAPI cufftDestroy(cufftHandle plan) { + ORT_CUFFT_FORWARD_RESULT(cufftDestroy, plan); +} + +cufftResult CUFFTAPI cufftSetStream(cufftHandle plan, cudaStream_t stream) { + ORT_CUFFT_FORWARD_RESULT(cufftSetStream, plan, stream); +} + +cufftResult CUFFTAPI cufftXtMakePlanMany(cufftHandle plan, int rank, long long int* n, + long long int* inembed, long long int istride, long long int idist, + cudaDataType inputtype, + long long int* onembed, long long int ostride, long long int odist, + cudaDataType outputtype, + long long int batch, size_t* workSize, cudaDataType executiontype) { + ORT_CUFFT_FORWARD_RESULT(cufftXtMakePlanMany, plan, rank, n, inembed, istride, idist, inputtype, onembed, ostride, + odist, outputtype, batch, workSize, executiontype); +} + +cufftResult CUFFTAPI cufftXtExec(cufftHandle plan, void* input, void* output, int direction) { + ORT_CUFFT_FORWARD_RESULT(cufftXtExec, plan, input, output, direction); +} + +} // extern "C" + +#undef ORT_CUFFT_FORWARD_RESULT + +#endif // USE_CUDA_MINIMAL diff --git a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h index 7c6cdd0e4dc80..ed698aaf74eb1 100644 --- a/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h +++ b/onnxruntime/core/providers/cuda/plugin/cuda_kernel_adapter.h @@ -34,6 +34,7 @@ #include #include "core/providers/cuda/shared_inc/cuda_call.h" #include "core/providers/cuda/cudnn_loader.h" +#include "core/providers/cuda/cufft_loader.h" #include "contrib_ops/cuda/bert/attention_kernel_options.h" #ifdef __CUDACC__ @@ -279,6 +280,11 @@ using ::onnxruntime::HandleNegativeAxis; { \ cufftResult _status = (expr); \ if (_status != CUFFT_SUCCESS) { \ + if (!onnxruntime::cuda::CufftLibrary::Get().Available()) { \ + return onnxruntime::common::Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::NOT_IMPLEMENTED, \ + std::string("cuFFT is unavailable for CUDA Plugin Execution Provider: ") + \ + onnxruntime::cuda::CufftLibrary::Get().Error()); \ + } \ return onnxruntime::common::Status(onnxruntime::common::ONNXRUNTIME, onnxruntime::common::FAIL, std::string("cuFFT error: ") + std::to_string((int)_status)); \ } \ }