ORT 1.28.0 release cherry-pick round 2

.github/workflows/linux_cuda_no_cudnn.yml

@@ -96,14 +96,15 @@ jobs:
             done < perms.txt
           fi
 
-      - name: Verify CUDA provider has no direct cuDNN dependency
+      - name: Verify CUDA provider has no direct cuDNN or cuFFT dependency
         run: |
           docker run --rm --gpus all \
             -v "${{ runner.temp }}/Release:/build/Release" \
             "${{ steps.build_docker_image_step.outputs.full-image-name }}" \
             bash -lc 'set -euo pipefail
           ldd /build/Release/Release/libonnxruntime_providers_cuda.so | tee /tmp/ldd.txt
-          ! grep -i cudnn /tmp/ldd.txt'
+          ! grep -i cudnn /tmp/ldd.txt
+          ! grep -i cufft /tmp/ldd.txt'
 
       - name: Run no-cuDNN CUDA EP op smoke test
         run: |
@@ -136,6 +137,24 @@ jobs:
             exit 1
           fi
 
+          # Likewise remove cuFFT so the smoke test proves the provider loads and runs
+          # non-FFT operators when cuFFT is physically unavailable (it is loaded lazily and
+          # only needed by the Rfft/Irfft contrib ops).
+          for root in /usr /opt /lib /lib64; do
+            if [ -e "$root" ]; then
+              find "$root" -name "libcufft*.so*" -print -delete 2>/dev/null || true
+            fi
+          done
+          ldconfig
+          if ldconfig -p | grep -qi cufft; then
+            echo "cuFFT remains available in the dynamic linker cache" >&2
+            exit 1
+          fi
+          if find /usr /opt /lib /lib64 -name "libcufft*.so*" -print -quit 2>/dev/null | grep -q .; then
+            echo "cuFFT runtime libraries remain in the no-cuDNN test container" >&2
+            exit 1
+          fi
+
           WHEEL_PATH=$(find /build/Release/Release/dist -type f -name "onnxruntime_gpu-*.whl" | head -n 1)
           if [ -z "$WHEEL_PATH" ]; then
             echo "No built onnxruntime GPU wheel found under /build/Release/Release/dist" >&2
@@ -254,5 +273,32 @@ jobs:
               np.argmin(data, axis=1).reshape(2, 1).astype(np.int64),
           )
 
-          print("CUDA no-cuDNN op smoke tests passed")
+          # Rfft requires cuFFT, which was physically removed above. With CPU fallback
+          # disabled the node must run on CUDA and fail cleanly with NOT_IMPLEMENTED
+          # (proving cuFFT is optional and its absence does not crash the provider).
+          rfft_model = helper.make_model(
+              helper.make_graph(
+                  [helper.make_node("Rfft", ["x"], ["y"], domain="com.microsoft", signal_ndim=1, onesided=1)],
+                  "no_cufft_rfft",
+                  [helper.make_tensor_value_info("x", TensorProto.FLOAT, [2, 4])],
+                    [helper.make_tensor_value_info("y", TensorProto.FLOAT, [2, 3, 2])],
+              ),
+              opset_imports=[helper.make_opsetid("", 17), helper.make_opsetid("com.microsoft", 1)],
+          )
+          rfft_model.ir_version = 10
+          rfft_options = ort.SessionOptions()
+          rfft_options.add_session_config_entry("session.disable_cpu_ep_fallback", "1")
+          try:
+              rfft_sess = ort.InferenceSession(
+                  rfft_model.SerializeToString(), sess_options=rfft_options, providers=providers
+              )
+              rfft_sess.run(None, {"x": np.random.rand(2, 4).astype(np.float32)})
+          except Exception as exc:  # noqa: BLE001
+              message = str(exc).lower()
+              assert "cufft" in message, "Rfft without cuFFT raised an unexpected error: " + str(exc)
+              print("[no-cuFFT] Rfft correctly failed: " + str(exc).splitlines()[0])
+          else:
+              raise AssertionError("Rfft unexpectedly succeeded while cuFFT was unavailable")
+
+          print("CUDA no-cuDNN/no-cuFFT op smoke tests passed")
           PY'

.github/workflows/windows_cuda_no_cudnn.yml

@@ -203,10 +203,20 @@ jobs:
           dir
         shell: pwsh
 
+      - name: Remove cuFFT runtime DLLs
+        shell: pwsh
+        run: |
+          $cudaRoot = Join-Path $env:RUNNER_TEMP "v13.0"
+          Get-ChildItem -Path $cudaRoot -Recurse -Include "cufft*.dll" | Remove-Item -Force
+          if (Get-ChildItem -Path $cudaRoot -Recurse -Include "cufft*.dll" -ErrorAction SilentlyContinue) {
+            Write-Error "cuFFT runtime DLLs must not be present in the no-cuDNN test environment"
+            exit 1
+          }
+
       - name: Add CUDA to PATH
         shell: pwsh
         run: |
-          Write-Host "Adding CUDA to PATH without adding any cuDNN directory"
+          Write-Host "Adding CUDA to PATH without cuDNN or cuFFT runtime DLLs"
           Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\bin"
           Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\bin\x64"
           Add-Content -Path $env:GITHUB_PATH -Value "$env:RUNNER_TEMP\v13.0\extras\CUPTI\lib64"
@@ -226,7 +236,7 @@ jobs:
         shell: pwsh
         run: nvidia-smi
 
-      - name: Verify CUDA plugin has no direct cuDNN dependency
+      - name: Verify CUDA plugin has no direct cuDNN or cuFFT dependency
         shell: pwsh
         run: |
           $pluginPath = "${{ runner.temp }}\build\Release\Release\onnxruntime_providers_cuda.dll"
@@ -240,6 +250,10 @@ jobs:
             Write-Error "CUDA plugin EP has a direct cuDNN dependency"
             exit 1
           }
+          if (Select-String -Path $env:RUNNER_TEMP\cuda_plugin_dependents.txt -Pattern "cufft" -SimpleMatch -Quiet) {
+            Write-Error "CUDA plugin EP has a direct cuFFT dependency"
+            exit 1
+          }
 
       - name: Run CUDA Plugin EP Python Tests without cuDNN
         working-directory: ${{ github.workspace }}\onnxruntime\test\python\transformers

cmake/onnxruntime_providers_cuda.cmake

@@ -402,7 +402,9 @@
       endif()
       target_compile_definitions(${target} PRIVATE NV_CUDNN_FRONTEND_USE_DYNAMIC_LOADING)
       target_include_directories(${target} PRIVATE ${CUDNN_INCLUDE_DIR})
-      target_link_libraries(${target} PRIVATE CUDA::cublasLt CUDA::cublas cudnn_frontend CUDA::curand CUDA::cufft CUDA::cudart CUDA::cuda_driver
+      # cuDNN and cuFFT are loaded dynamically at runtime (see cudnn_stub.cc / cufft_stub.cc), so they are
+      # intentionally not linked here to avoid a hard runtime dependency when the related ops are not used.
+      target_link_libraries(${target} PRIVATE CUDA::cublasLt CUDA::cublas cudnn_frontend CUDA::curand CUDA::cudart CUDA::cuda_driver
               ${ABSEIL_LIBS} ${ONNXRUNTIME_PROVIDERS_SHARED} Boost::mp11 safeint_interface)
     endif()
 

cmake/onnxruntime_providers_cuda_plugin.cmake

@@ -393,11 +393,13 @@ onnxruntime_add_include_to_target(
 add_dependencies(onnxruntime_providers_cuda_plugin ${onnxruntime_EXTERNAL_DEPENDENCIES})
 
 # Link libraries
+# cuDNN and cuFFT are loaded dynamically at runtime (see cudnn_stub.cc / cufft_stub.cc, which are
+# picked up by the GLOB above), so they are intentionally not linked here to avoid a hard runtime
+# dependency when the related ops are not used.
 target_link_libraries(onnxruntime_providers_cuda_plugin PRIVATE
     CUDA::cudart
     CUDA::cublas
     CUDA::cublasLt
-    CUDA::cufft
     CUDA::cuda_driver
     cudnn_frontend
     Boost::mp11

cmake/onnxruntime_unittests.cmake

@@ -1051,6 +1051,7 @@ if (onnxruntime_ENABLE_CUDA_EP_INTERNAL_TESTS AND onnxruntime_BUILD_CUDA_EP_AS_P
       "${ONNXRUNTIME_ROOT}/core/providers/cuda/cuda_utils.cu"
       "${ONNXRUNTIME_ROOT}/core/providers/cuda/cudnn_common.cc"
       "${ONNXRUNTIME_ROOT}/core/providers/cuda/cudnn_loader.cc"
+      "${ONNXRUNTIME_ROOT}/core/providers/cuda/cufft_loader.cc"
       "${ONNXRUNTIME_ROOT}/core/providers/cuda/reduction/reduction_functions.cc"
       "${ONNXRUNTIME_ROOT}/core/providers/cuda/reduction/reduction_functions.cu"
       "${TEST_SRC_DIR}/providers/cuda/test_cases/cuda_plugin_test_shims.cc"
@@ -1408,7 +1409,6 @@ block()
       CUDA::cublas
       CUDA::cublasLt
       CUDA::curand
-      CUDA::cufft
       CUDA::cuda_driver
       CUDNN::cudnn
       cudnn_frontend)

csharp/ApiDocs/ApiDocs.csproj

@@ -6,12 +6,15 @@
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
   </PropertyGroup>
 
-  <ItemGroup>
-    <PackageReference Include="docfx.console" Version="2.59.4">
-      <PrivateAssets>all</PrivateAssets>
-      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
-    </PackageReference>
-  </ItemGroup>
+  <!--
+    Documentation is generated by the maintained 'docfx' .NET global tool, not by
+    the legacy 'docfx.console' NuGet package. docfx.console is deprecated (last
+    release 2.59.4) and injected MSBuild targets that ran an outdated docfx.exe
+    during the solution build, which breaks against current MSBuild/VS. Doc
+    generation is now invoked explicitly via the 'docfx' global tool
+    (see .github/workflows/publish-csharp-apidocs.yml and the C# packaging
+    pipeline), so building OnnxRuntime.CSharp.sln no longer runs docfx.
+  -->
 
   <ItemGroup>
     <Folder Include="images\" />

docs/CUDA_cuDNN_Optional_Design.md

@@ -358,6 +358,40 @@ TensorRT / NV‑RTX EPs are untouched and continue to link cuDNN as before. (If
 and the shimmed CUDA EP are in the same process, symbol collision must be avoided — see
 §8 Risks.)
 
+### 3.6 Applying the same pattern to cuFFT
+
+The identical shim + lazy‑loader technique is used to drop the hard dependency on **cuFFT**
+(`libcufft.so.*` / `cufft64_*.dll`). cuFFT is only needed by the FFT contrib operators
+(`Rfft` / `Irfft` in `contrib_ops/cuda/math`), which use a small, enumerable set of entry
+points: `cufftCreate`, `cufftDestroy`, `cufftSetStream`, `cufftXtMakePlanMany`, and
+`cufftXtExec`.
+
+Implementation:
+
+- **Loader** — `onnxruntime/core/providers/cuda/cufft_loader.{h,cc}` defines a
+  `CufftLibrary` singleton that mirrors `CudnnLibrary`: it lazily `dlopen`/`LoadLibrary`s the
+  cuFFT runtime, resolves symbols on demand, caches them, and exposes `Available()` / `Error()`.
+  It uses the same security‑conscious search behavior as the cuDNN loader (Windows
+  `LOAD_LIBRARY_SEARCH_DEFAULT_DIRS` plus a PATH fallback that never loads from the current
+  working directory). The candidate library name is selected at **compile time** from the
+  `CUDA_VERSION` macro (cuFFT 12 for CUDA 13.x, cuFFT 11 for CUDA 12.x), because cuFFT's
+  SONAME/DLL version tracks the CUDA major version. Unsupported CUDA major versions fail at
+  compile time until their cuFFT library mapping is added explicitly.
+- **Shim** — `onnxruntime/core/providers/cuda/cufft_stub.cc` defines the five entry points
+  above as trampolines that forward to the resolved symbol (returning `CUFFT_INTERNAL_ERROR`
+  when the library is unavailable), so ORT's direct calls link against *our* definitions and
+  the final binary has **no import entry for libcufft**.
+- **Availability guard** — `CudaCall<cufftResult>` (in `cuda_call.cc`) and the plugin's
+  `CUFFT_RETURN_IF_ERROR` macro check `CufftLibrary::Available()` and return a clear
+  `NOT_IMPLEMENTED` status ("cuFFT is unavailable …") when cuFFT is missing, exactly like the
+  cuDNN path.
+
+Unlike cuDNN, cuFFT needs **no provider option** (there is no `enable_cufft`): it is always
+loaded lazily on first FFT‑op use, and there is no `cufft_frontend` equivalent to configure.
+cuFFT headers (part of the CUDA toolkit) are still required at build time, but `CUDA::cufft`
+is no longer linked in `onnxruntime_providers_cuda.cmake`,
+`onnxruntime_providers_cuda_plugin.cmake`, or the CUDA unit‑test target.
+
 ---
 
 ## 4. Phase 1 — Make cuDNN optional (no new kernels)

docs/contrib_ops/cuda/matmul_nbits.md

@@ -307,6 +307,19 @@ present. `ComputeInternal` then:
 
 ## 9. Testing
 
+- CUDA EP internal tests run through `CUDA_EP_Unittest` in
+  [onnxruntime/test/providers/cuda/cuda_provider_test.cc](../../../onnxruntime/test/providers/cuda/cuda_provider_test.cc).
+  Run them from `onnxruntime_provider_test` with:
+
+  ```bash
+  ./onnxruntime_provider_test --gtest_filter=CUDA_EP_Unittest.*
+  ```
+
+  This wrapper executes the internal CUDA-UT shared library and covers the
+  fpA_intB / MatMulNBits groupwise GEMM tests under
+  [onnxruntime/test/contrib_ops/cuda_kernels/fpA_intB_gemm_kernel_test.cc](../../../onnxruntime/test/contrib_ops/cuda_kernels/fpA_intB_gemm_kernel_test.cc)
+  as well as the SM90 validation tests in
+  [onnxruntime/test/contrib_ops/cuda_kernels/matmul_nbits_sm90_validation_test.cc](../../../onnxruntime/test/contrib_ops/cuda_kernels/matmul_nbits_sm90_validation_test.cc).
 - Python operator tests: `onnxruntime/test/python/transformers` (see the QMoE /
   GEMV profiling helpers, e.g. `profile_qmoe_gemv.sh`).
 - CUDA prepacked-weight parity tests:

docs/cuda_plugin_ep/cuda_plugin_ep_design.md

@@ -43,6 +43,12 @@ There is intentionally no provider option for a custom cuDNN DLL/SO path. Provid
 
 No-cuDNN plugin validation runs `test_cuda_plugin_ep.py` with `ORT_TEST_CUDA_PLUGIN_EP=1` and `ORT_TEST_CUDA_PLUGIN_NO_CUDNN=1`. That mode passes `enable_cudnn=0` to plugin sessions and skips tests for operators that still require cuDNN in the current implementation. Non-cuDNN operator coverage, plugin registration, device enumeration, graph assignment, CUDA graph, I/O binding, and profiling tests continue to run.
 
+### 2.1.2 Optional cuFFT Runtime Dependency
+
+The CUDA Plugin EP applies the same optional-dependency model to **cuFFT**. cuFFT is only used by the FFT contrib operators (`Rfft` / `Irfft`), so the plugin shared library must not link directly to cuFFT or contain a cuFFT DLL/SO in its dynamic dependency table. cuFFT is loaded lazily through the ORT cuFFT loader (`CufftLibrary`, `core/providers/cuda/cufft_loader.{h,cc}`) on first FFT-op use; the entry points ORT calls are provided by `cufft_stub.cc` trampolines that forward to the resolved symbols.
+
+Unlike cuDNN, there is **no provider option** for cuFFT (no `enable_cufft`) and no frontend library to configure. When cuFFT is unavailable, `CUFFT_RETURN_IF_ERROR` (and `CudaCall<cufftResult>`) return `NOT_IMPLEMENTED` with an actionable message, so FFT ops fail cleanly while all other operators keep working. cuFFT headers (part of the CUDA toolkit) are still required at build time, but `CUDA::cufft` is no longer linked. The loader picks the cuFFT library name at compile time from the `CUDA_VERSION` macro (cuFFT 12 for CUDA 13.x, cuFFT 11 for CUDA 12.x); unsupported CUDA major versions fail at compile time until their mapping is added.
+
 ### 2.2 Preprocessor Defines
 
 Each build target uses different preprocessor defines that control how framework types are resolved:
@@ -651,7 +657,7 @@ Use `onnxruntime_BUILD_CUDA_EP_AS_PLUGIN=OFF` when you need to build the origina
 
 ### 9.3 Plugin Independence
 
-The plugin build's `libonnxruntime_providers_cuda.so` is **fully self-contained**. It does not depend on `libonnxruntime_providers_shared.so` at load time. It statically links against `onnxruntime_framework`, `onnxruntime_graph`, `onnxruntime_common`, `onnxruntime_mlas`, `onnxruntime_flatbuffers`, and links dynamically against CUDA (`cudart`, `cublas`, `cublasLt`, `cufft`) and protobuf. cuDNN is loaded lazily only when enabled and available at runtime. Communication with the ORT runtime happens exclusively through the C API (`OrtApi`/`OrtEpApi`) passed at load time.
+The plugin build's `libonnxruntime_providers_cuda.so` is **fully self-contained**. It does not depend on `libonnxruntime_providers_shared.so` at load time. It statically links against `onnxruntime_framework`, `onnxruntime_graph`, `onnxruntime_common`, `onnxruntime_mlas`, `onnxruntime_flatbuffers`, and links dynamically against CUDA (`cudart`, `cublas`, `cublasLt`) and protobuf. cuDNN and cuFFT are loaded lazily only when available at runtime (cuDNN additionally gated by `enable_cudnn`), so neither appears in the library's dynamic dependency table. Communication with the ORT runtime happens exclusively through the C API (`OrtApi`/`OrtEpApi`) passed at load time.
 
 ### 9.4 Build Outputs
 

onnxruntime/contrib_ops/cuda/quantization/matmul_nbits.cc

@@ -18,6 +18,7 @@
 #include "contrib_ops/cuda/llm/fpA_intB_gemm_adaptor.h"
 #include "contrib_ops/cuda/llm/fpA_intB_gemm_preprocessors.h"
 #include "contrib_ops/cuda/llm/common/cuda_runtime_utils.h"
+#include "contrib_ops/cuda/quantization/matmul_nbits_sm90_validation.h"
 #endif
 #include "contrib_ops/cuda/llm/common/logger.h"
 #include "contrib_ops/cpu/quantization/matmul_nbits_helper.h"
@@ -56,6 +57,38 @@ int64_t MatMulNBits<T>::RequiredWeightPrepackedFormat() const {
   return FpAIntBPackingSmForKernel() == 90 ? kMatMulNBitsWeightPrepackedSm90 : kMatMulNBitsWeightPrepackedSm80;
 }
 
+// See matmul_nbits_sm90_validation.h for why these two functions are declared there but defined
+// here (non-inline, single definition, inside the CUDA EP translation unit that observes
+// COMPILE_HOPPER_TMA_GEMMS).
+bool IsNativeSm90FpAIntBGemmCompiled() {
+#if defined(COMPILE_HOPPER_TMA_GEMMS)
+  return true;
+#else
+  return false;
+#endif
+}
+
+void ValidateSm90PrepackedWeightSupport(int sm, int64_t block_size) {
+  // The native SM90 (Hopper TMA/WGMMA) mixed-GEMM kernel requires a compute-capability 9.0
+  // device and a block_size that is a multiple of the Hopper K tile (128 / sizeof(half) = 64).
+  // block_size=32 is only supported by the SM80/Ampere-class kernel + GEMV path.
+  ORT_ENFORCE(sm == 90,
+              "weight_prepacked=2 (SM90 layout) requires a compute capability 9.0 (Hopper) device, but got sm ", sm);
+#if !defined(COMPILE_HOPPER_TMA_GEMMS)
+  // The native SM90 (Hopper) fpA_intB TMA/WGMMA kernel is not compiled in this build (for
+  // example Windows/MSVC, where CUDA 13 NVCC host stubs hit MSVC C2719 with over-aligned TMA
+  // parameters; see docs/contrib_ops/cuda/moe_qmoe.md section 14.1). The SM90 weight layout
+  // cannot be consumed by the SM80 kernel, so fail early here with a clear message instead of
+  // dispatching to the throwing launcher stub during tactic profiling.
+  ORT_THROW(
+      "weight_prepacked=2 (SM90 layout) is not supported by this ONNX Runtime build "
+      "(the native SM90 Hopper fpA_intB kernel is unavailable, e.g. on Windows/MSVC). "
+      "Re-export the model with weight_prepacked=0 or 1 to use the SM80-compatible fpA_intB layout.");
+#endif
+  ORT_ENFORCE(block_size == 64 || block_size == 128,
+              "weight_prepacked=2 (SM90 layout) supports block_size 64 or 128 only, but got ", block_size);
+}
+
 template <typename T>
 void MatMulNBits<T>::InitGemmProfiler(int sm) {
   gemmProfiler_ = s_profilerManager.createGemmPluginProfiler(/*inference*/ false);