1.25.0 rc

.coderabbit.yaml

@@ -0,0 +1,14 @@
+# Repository-specific override
+reviews:
+  profile: chill
+  auto_review:
+    enabled: true # Enable auto-review for this repository
+  path_instructions:
+    - path: "python/cudnn/**"
+      instructions: |
+        - Focus on documentation.
+        - Focus on whether there are test cases in test/python/fe_api
+    - path: "include/**"
+      instructions: |
+        - Make sure the cpp code has good quality
+        - Make sure the dynamic and static cudnn versions are met.

.pre-commit-config.yaml

@@ -0,0 +1,29 @@
+repos:
+-   repo: https://github.com/pre-commit/mirrors-clang-format
+    rev: v21.1.6
+    hooks:
+    -   id: clang-format
+        types_or: [c++, c, cuda]
+-   repo: https://github.com/psf/black-pre-commit-mirror
+    rev: 26.3.1
+    hooks:
+    -   id: black
+        name: black
+        description: "Black: The uncompromising Python code formatter"
+        entry: black
+        language: python
+        minimum_pre_commit_version: 2.9.2
+        require_serial: true
+        types_or: [python, pyi]
+        args: ["--line-length", "160"]
+    -   id: black-jupyter
+        name: black-jupyter
+        description:
+            "Black: The uncompromising Python code formatter (with Jupyter Notebook support)"
+        entry: black
+        language: python
+        minimum_pre_commit_version: 2.9.2
+        require_serial: true
+        types_or: [python, pyi, jupyter]
+        additional_dependencies: [".[jupyter]"]
+        args: ["--line-length", "160"]

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.23)
 
-project(cudnn_frontend VERSION 1.24.1)
+project(cudnn_frontend VERSION 1.25.0)
 
 option(CUDNN_FRONTEND_SKIP_JSON_LIB "Defines whether FE should not include nlohmann/json.hpp." OFF)
 option(CUDNN_FRONTEND_BUILD_SAMPLES "Defines if samples are built or not." ON)

README.md

@@ -40,7 +40,7 @@ We are now shipping **OSS kernels**, allowing you to inspect, modify, and contri
 *   **[cudnn SDPA Fprop](https://github.com/NVIDIA/cudnn-frontend/tree/main/include/cudnn_frontend/generated/sdpa):** Open sourcing the Hopper and Blackwell fprop kernels with stats.
 *   **[Fused RMSNorm + SiLU](https://github.com/NVIDIA/cudnn-frontend/tree/main/include/cudnn_frontend/generated/rms_norm_silu):** Implementation of a fused kernel of RMS normalization followed by SiLU (Swish) activation.
 *   **[SDPA PyTorch Op](https://github.com/NVIDIA/cudnn-frontend/tree/main/python/cudnn/experimental/ops):** PyTorch custom operator for cuDNN-accelerated Scaled Dot-Product Attention with autograd and `torch.compile` support.
-*   **[DSA](https://github.com/NVIDIA/cudnn-frontend/tree/main/python/cudnn/native_sparse_attention):** DSA/CSA kernels for DSv4 and DSv3.2 for fprop and bprop.
+*   **[DSA](https://github.com/NVIDIA/cudnn-frontend/tree/main/python/cudnn/deepseek_sparse_attention):** DSA/CSA kernels for DSv4 and DSv3.2 for fprop and bprop.
 
 ## Tech talks
 
@@ -152,6 +152,24 @@ export CUDNN_FRONTEND_LOG_FILE=execution_log.txt
 
 Alternatively, you can control logging programmatically via `cudnn_frontend::isLoggingEnabled()`.
 
+### Overriding the CUDA runtime library
+
+When the frontend is built with dynamic loading enabled, it locates the CUDA runtime
+(`libcudart.so.*`) at runtime by searching for the supported major versions. In some
+environments (for example, containers such as GKE where the TCPXO NCCL plugin mounts a
+different `libcudart` major version from the host) multiple versions of `libcudart` may be
+visible on the library search path, and the automatic detection aborts with a
+`Multiple libcudart libraries found` error.
+
+To resolve this, set the `CUDNN_FRONTEND_CUDART_LIB_NAME` environment variable to the
+library name (or full path) that should be loaded. This bypasses the automatic detection:
+
+```bash
+export CUDNN_FRONTEND_CUDART_LIB_NAME=libcudart.so.13
+# or an absolute path
+export CUDNN_FRONTEND_CUDART_LIB_NAME=/usr/local/cuda/lib64/libcudart.so.13
+```
+
 ## License
 
 This project is licensed under the [MIT License](LICENSE).

benchmark/sdpa_benchmark_training/ACKNOWLEDGEMENTS.md

@@ -0,0 +1,68 @@
+# Acknowledgements
+
+## Flash-Attention and Tri Dao
+
+cuDNN's attention stack has been shaped by the Flash-Attention project.
+Many of the techniques that define a modern fused, IO-aware attention
+kernel — the online-softmax recurrence, the tile schedule, the
+warp-specialized producer / consumer split, the asynchronous epilogue —
+were articulated and shipped early by Tri Dao and the Flash-Attention
+contributors, and cuDNN's SDPA engines have benefited from studying their
+kernels and numerics across successive generations.
+
+We work in close collaboration with Tri Dao and the Flash-Attention project,
+and we are grateful for the openness with which the techniques, the kernels,
+and the rationale behind them have been shared. Many of the optimizations
+that make cuDNN attention competitive today were absorbed directly from that
+work, and were possible only because the underlying ideas — and the reference
+implementations — were available to study, profile, and adapt. The current
+generation of cuDNN attention reaches rough performance parity with
+Flash-Attention 4, and that parity is, in large part, a reflection of how
+much of FAv4's thinking now lives inside cuDNN.
+
+This is a collaboration we value, and one we hope to keep deepening.
+
+## Techniques absorbed from Flash-Attention 4
+
+The following list is not meant for a landing page — we surface it here for
+anyone who asks where specific ideas in cuDNN attention came from. Each of
+these originated in (or was first shipped by) the Flash-Attention project,
+and was subsequently adapted inside cuDNN across architectures (Ampere
+through Blackwell) and across data types (fp16, bf16, fp8):
+
+- **Rescale-threshold skipping.** Avoiding the per-tile output rescale when
+  the running-max update is small enough that the correction factor is
+  numerically a no-op. Adapted to cuDNN's softmax recurrence and extended
+  to additional dtypes and tile shapes.
+- **Skip correction.** Elision of the second-pass correction when a tile
+  cannot move the running max — applied across architectures.
+- **MUFU exp2 emulation.** The fast-path approximation of `exp2` used to
+  hide MUFU latency in the softmax inner loop, re-implemented across
+  architectures and dtypes.
+
+In each case the core insight is the Flash-Attention team's; cuDNN's
+contribution is the porting, generalization, and integration into the
+heuristics, code generation, and graph layer that ship in the library.
+
+## Citation
+
+If you use cuDNN's SDPA / Flash-Attention engines in your work, please cite
+the foundational Flash-Attention papers in addition to any cuDNN reference:
+
+```bibtex
+@inproceedings{dao2022flashattention,
+  title={Flash{A}ttention: Fast and Memory-Efficient Exact Attention with {IO}-Awareness},
+  author={Dao, Tri and Fu, Daniel Y. and Ermon, Stefano and Rudra, Atri and R{\'e}, Christopher},
+  booktitle={Advances in Neural Information Processing Systems (NeurIPS)},
+  year={2022}
+}
+
+@inproceedings{dao2023flashattention2,
+  title={Flash{A}ttention-2: Faster Attention with Better Parallelism and Work Partitioning},
+  author={Dao, Tri},
+  booktitle={International Conference on Learning Representations (ICLR)},
+  year={2024}
+}
+```
+
+Upstream source: <https://github.com/Dao-AILab/flash-attention>.

benchmark/sdpa_benchmark_training/README.md

@@ -15,7 +15,7 @@ This directory contains benchmarking tools for Scaled Dot Product Attention (SDP
   - `wan22.py` - Wan 2.2 A14B video DiT self-attention benchmarks (bidirectional, no mask)
   - `ltx2.py` - LTX-2 video DiT self-attention benchmarks (bidirectional, no mask)
   - `gpt_oss.py` - GPT-OSS sliding-window-attention GQA benchmarks (causal, SWA=128)
-  - `qwen35.py` - Qwen 3.5 GQA benchmarks (head_dim=256, causal, bf16 fwd-only — Blackwell bwd/fp8/fa4 limits)
+  - `qwen35.py` - Qwen 3.5 GQA benchmarks (head_dim=256, causal, bf16 bidirectional — Blackwell fp8/fa4 limits)
   - `auto_regressive_dit.py` - Autoregressive video DiT (short Q, long cached KV, bf16/mxfp8, no_mask)
 - `runner.py` - Configuration-based benchmark runner
 - `config_types.py` - Data types for benchmark configuration
@@ -53,7 +53,7 @@ python -m benchmark.sdpa_benchmark_training.runner --config wan22
 # Run LTX-2 benchmark suite
 python -m benchmark.sdpa_benchmark_training.runner --config ltx2
 
-# Run Qwen 3.5 benchmark suite (cuDNN bf16 fwd only at head_dim=256)
+# Run Qwen 3.5 benchmark suite (cuDNN bf16 at head_dim=256)
 python -m benchmark.sdpa_benchmark_training.runner --config qwen35
 
 # Run Autoregressive video DiT benchmark suite (short Q, long cached KV)
@@ -305,7 +305,7 @@ results/<config>/<gpu>/
     <config>_<mask>_det_overhead.png     # bwd bf16: det vs non-det comparison
 ```
 
-Runs were captured on Lyris GB200 and GB300 with cuDNN 9.22.0 and FAv4 4.0.0b10.
+Runs were captured on GB200 and GB300 with cuDNN 9.23.0 and FAv4 4.0.0b15.
 
 ### GB300 - Llama 3.1 Causal (top_left)
 ![Llama 3.1 Causal on GB300](results/llama3.1/gb300/llama3.1_top_left.png)
@@ -337,9 +337,19 @@ Runs were captured on Lyris GB200 and GB300 with cuDNN 9.22.0 and FAv4 4.0.0b10.
 ![GPT-OSS Causal on GB300](results/gpt_oss/gb300/gpt_oss_top_left.png)
 - `batch=2; num_q_heads=128; num_kv_heads=128; head_dim=64; sliding_window_size=128`
 
-### GB300 - Qwen 3.5 (head_dim=256, fwd only)
+### GB300 - Qwen 3.5 (head_dim=256)
 ![Qwen 3.5 Causal on GB300](results/qwen35/gb300/qwen35_top_left.png)
-- `batch=2; num_q_heads=32; num_kv_heads=2; head_dim=256` — cuDNN BF16 fwd only at head_dim=256 on Blackwell
+- `batch=2; num_q_heads=32; num_kv_heads=2; head_dim=256` — cuDNN BF16 at head_dim=256 on Blackwell
+
+### GB300 - Autoregressive video DiT (short Q, long cached KV)
+![Autoregressive DiT on GB300](results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png)
+- `batch=1; num_q_heads=9; num_kv_heads=9; head_dim=128; s_q ∈ {985..8192}; s_kv=62208`
+- Forward-only (autoregressive inference). cuDNN 9.30.0 with prefill split-K on bf16/fp8/mxfp8; FAv4 BF16 swept over `num_splits ∈ {1, 2, 4, 8, 16, 32}` with the best annotated on each bar (`ks=`). FAv4 FP8/MXFP8 are absent — the CuTe-DSL FAv4 build rejects those input types.
+- Reproduce with `python -m benchmark.sdpa_benchmark_training.bench_ar_dit_peak --out <path>`.
+
+### GB200 - Autoregressive video DiT
+![Autoregressive DiT on GB200](results/auto_regressive_dit/gb200/auto_regressive_dit_no_mask.png)
+- Same configuration as the GB300 chart above, captured on GB200.
 
 ### GB300 - Autoregressive video DiT (short Q, long cached KV)
 ![Autoregressive DiT on GB300](results/auto_regressive_dit/gb300/auto_regressive_dit_no_mask.png)

benchmark/sdpa_benchmark_training/bench_ar_dit_peak.py

@@ -9,6 +9,7 @@
 plus an extra ``num_splits`` column so the per-seqlen winners are
 visible.
 """
+
 from __future__ import annotations
 
 import argparse
@@ -73,12 +74,19 @@ def main():
             try:
                 r = run("cudnn", dt, s_q)
                 log.info(f"cudnn {dt:>8} s_q={s_q:>5}: {r['time_ms']:.3f} ms  {r['tflops']:.1f} TF")
-                rows.append(dict(
-                    backend="cudnn", data_type=dt, q_seqlen=s_q, kv_seqlen=S_KV,
-                    num_splits=0,
-                    time_ms=r["time_ms"], tflops=r["tflops"],
-                    gpu_name=gpu_name, cudnn_backend_version=cudnn_be,
-                ))
+                rows.append(
+                    dict(
+                        backend="cudnn",
+                        data_type=dt,
+                        q_seqlen=s_q,
+                        kv_seqlen=S_KV,
+                        num_splits=0,
+                        time_ms=r["time_ms"],
+                        tflops=r["tflops"],
+                        gpu_name=gpu_name,
+                        cudnn_backend_version=cudnn_be,
+                    )
+                )
             except Exception as e:
                 log.info(f"cudnn {dt} s_q={s_q} FAILED: {e}")
 
@@ -89,21 +97,27 @@ def main():
                 r = run("flash_attention_4", "bfloat16", s_q, fa4_num_splits=ks)
                 log.info(f"  fa4  bfloat16 s_q={s_q:>5} num_splits={ks:>2}: {r['time_ms']:.3f} ms  {r['tflops']:.1f} TF")
                 if best is None or r["time_ms"] < best["time_ms"]:
-                    best = dict(r); best["num_splits"] = ks
+                    best = dict(r)
+                    best["num_splits"] = ks
             except Exception as e:
                 log.info(f"  fa4  bfloat16 s_q={s_q} num_splits={ks} FAILED: {e}")
         if best is not None:
             log.info(f"fa4  bfloat16 s_q={s_q:>5} BEST num_splits={best['num_splits']:>2}: {best['time_ms']:.3f} ms  {best['tflops']:.1f} TF")
-            rows.append(dict(
-                backend="flash_attention_4", data_type="bfloat16",
-                q_seqlen=s_q, kv_seqlen=S_KV,
-                num_splits=best["num_splits"],
-                time_ms=best["time_ms"], tflops=best["tflops"],
-                gpu_name=gpu_name, cudnn_backend_version=cudnn_be,
-            ))
-
-    fields = ["backend", "data_type", "q_seqlen", "kv_seqlen", "num_splits",
-              "time_ms", "tflops", "gpu_name", "cudnn_backend_version"]
+            rows.append(
+                dict(
+                    backend="flash_attention_4",
+                    data_type="bfloat16",
+                    q_seqlen=s_q,
+                    kv_seqlen=S_KV,
+                    num_splits=best["num_splits"],
+                    time_ms=best["time_ms"],
+                    tflops=best["tflops"],
+                    gpu_name=gpu_name,
+                    cudnn_backend_version=cudnn_be,
+                )
+            )
+
+    fields = ["backend", "data_type", "q_seqlen", "kv_seqlen", "num_splits", "time_ms", "tflops", "gpu_name", "cudnn_backend_version"]
     out = Path(args.out)
     out.parent.mkdir(parents=True, exist_ok=True)
     with out.open("w", newline="") as f:

benchmark/sdpa_benchmark_training/benchmark_single_sdpa.py

@@ -121,8 +121,7 @@ def parse_args():
         "--fa4_num_splits",
         default=None,
         type=int,
-        help="FlashAttention-4 only: force num_splits (KV split count). "
-             "Default is None (FA4 picks automatically).",
+        help="FlashAttention-4 only: force num_splits (KV split count). " "Default is None (FA4 picks automatically).",
     )
     parser.add_argument(
         "--fwd_bwd",

benchmark/sdpa_benchmark_training/charts.py

@@ -134,7 +134,6 @@ def generate_charts_by_mask(
     import seaborn as sns
 
     df = df[df["success"] == True].copy()
-    df = df[df["backend"] != "flash_attention_4"].copy()
 
     # Main charts show only non-deterministic mode; det-vs-nondet comparison
     # is in generate_det_overhead_charts. Fwd rows always carry det=False, so

benchmark/sdpa_benchmark_training/configs/qwen35.py

@@ -36,7 +36,7 @@
     # fa4 rows are kept (they fail) to document the sm100 kernel limitation.
     data_types=["bfloat16"],
     attn_masks=["top_left"],  # Causal only
-    profile_pass="fwd",  # Forward-only (bwd blocked at head_dim=256)
+    profile_pass="both",
     deterministic_bwd=[False, True],
     batch_size=1,
     num_iterations=10,