From 446c569276a184efb12ac9a347ae7a98915b140a Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 17:16:14 +0000 Subject: [PATCH 01/11] Bump nvidia-cutlass-dsl to >=4.5.0 for SM121 support CUTLASS DSL 4.5 adds sm_121a to MmaSM120BlockScaledOp.admissible_archs, fixing FP4 block-scaled MMA on DGX Spark (SM121) without a monkey-patch. Also update the cute-dsl test skip condition to run on SM120/SM121. Co-authored-by: Cursor --- pyproject.toml | 7 +++---- tests/gemm/test_mm_fp4.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a854c1de4b..6dc828fb3b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,15 +17,14 @@ name = "flashinfer-python" description = "FlashInfer: Kernel Library for LLM Serving" requires-python = ">=3.10,<4.0" authors = [{ name = "FlashInfer team" }] -license = "Apache-2.0" +license = {text = "Apache-2.0"} readme = "README.md" urls = { Homepage = "https://github.com/flashinfer-ai/flashinfer" } dynamic = ["dependencies", "version"] -license-files = ["LICENSE", "LICENSE*.txt"] [project.optional-dependencies] -cu12 = ["nvidia-cutlass-dsl>=4.4.2"] -cu13 = ["nvidia-cutlass-dsl[cu13]>=4.4.2"] +cu12 = ["nvidia-cutlass-dsl>=4.5.0"] +cu13 = ["nvidia-cutlass-dsl[cu13]>=4.5.0"] [project.scripts] flashinfer = "flashinfer.__main__:cli" diff --git a/tests/gemm/test_mm_fp4.py b/tests/gemm/test_mm_fp4.py index 519536ae0b..70eace91b1 100644 --- a/tests/gemm/test_mm_fp4.py +++ b/tests/gemm/test_mm_fp4.py @@ -32,8 +32,8 @@ def _test_mm_fp4( if backend == "cute-dsl": if not use_128x4_sf_layout: pytest.skip("cute_dsl backend only supports 128x4 SF layout") - if compute_capability[0] not in [10]: - pytest.skip("cute_dsl backend only supports SM100/SM103 GPUs.") + if compute_capability[0] not in [10, 12]: + pytest.skip("cute_dsl backend only supports SM100/SM103/SM120/SM121 GPUs.") if backend == "b12x": if not use_128x4_sf_layout: pytest.skip("b12x backend only supports 128x4 SF layout") From 49347ad8b3ee4368623c6ceb48c002e7c6c60f8c Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 17:22:01 +0000 Subject: [PATCH 02/11] Bump nvidia-cutlass-dsl to >=4.5.0 in CI dependencies Update requirements.txt and Docker install script to match the pyproject.toml bump so CI picks up the new version. Co-authored-by: Cursor --- docker/install/install_python_packages.sh | 2 +- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/install/install_python_packages.sh b/docker/install/install_python_packages.sh index e1d91d8b97..6c980701b6 100644 --- a/docker/install/install_python_packages.sh +++ b/docker/install/install_python_packages.sh @@ -32,7 +32,7 @@ pip3 install responses pytest scipy build cuda-python nvshmem4py-cu12 if [[ "$CUDA_VERSION" == *"cu13"* ]]; then pip3 install --upgrade cuda-python==13.0 pip3 install --upgrade nvidia-cudnn-cu13 - pip3 install --upgrade "nvidia-cutlass-dsl[cu13]>=4.4.2" + pip3 install --upgrade "nvidia-cutlass-dsl[cu13]>=4.5.0" else pip3 install --upgrade cuda-python==12.* pip3 install --upgrade nvidia-cudnn-cu12 diff --git a/requirements.txt b/requirements.txt index 7eb97a4ab9..c7534c8108 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ einops ninja numpy nvidia-cudnn-frontend>=1.13.0 -nvidia-cutlass-dsl>=4.4.2 +nvidia-cutlass-dsl>=4.5.0 nvidia-ml-py packaging>=24.2 requests From 359d7d9e0ab17ef0e5438822f5d19fb33568b292 Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 17:24:41 +0000 Subject: [PATCH 03/11] a --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6dc828fb3b..19ae29c3c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,10 +17,10 @@ name = "flashinfer-python" description = "FlashInfer: Kernel Library for LLM Serving" requires-python = ">=3.10,<4.0" authors = [{ name = "FlashInfer team" }] -license = {text = "Apache-2.0"} +license = "Apache-2.0" readme = "README.md" urls = { Homepage = "https://github.com/flashinfer-ai/flashinfer" } -dynamic = ["dependencies", "version"] +license-files = ["LICENSE", "LICENSE*.txt"] [project.optional-dependencies] cu12 = ["nvidia-cutlass-dsl>=4.5.0"] From 3f337764e5322cfc710591585811c2a478319795 Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 17:48:51 +0000 Subject: [PATCH 04/11] Add requirements.txt sync to CI test setup The CI test runner uses `pip install -e . --no-deps`, which skips dependency resolution. Add an explicit `pip install -r requirements.txt` before the editable install so that dependency bumps on the branch (e.g. nvidia-cutlass-dsl) are picked up without rebuilding the Docker image. Co-authored-by: Cursor --- scripts/test_utils.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/test_utils.sh b/scripts/test_utils.sh index 9a2874fe58..d6a6d2c866 100755 --- a/scripts/test_utils.sh +++ b/scripts/test_utils.sh @@ -165,6 +165,9 @@ install_and_verify() { # Install precompiled kernels if enabled install_precompiled_kernels + # Sync dependencies from the branch's requirements.txt + pip install -r requirements.txt + # Install local python sources pip install -e . -v --no-deps echo "" From c0b88a9f0bd9d8fe8a0e4a98f270248c50c8d471 Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 17:52:57 +0000 Subject: [PATCH 05/11] Enable b12x FP4 tests on SM121 (CUTLASS DSL 4.5 adds sm_121a support) nvidia-cutlass-dsl 4.5.0 adds sm_121a to MmaSM120BlockScaledOp.admissible_archs, enabling warp-level MMA on SM121 (DGX Spark). Remove the SM121 test skips that were guarding against the missing arch in 4.4.2. Co-authored-by: Cursor --- tests/gemm/test_mm_fp4.py | 8 ++++---- tests/moe/test_b12x_fused_moe.py | 17 ----------------- 2 files changed, 4 insertions(+), 21 deletions(-) diff --git a/tests/gemm/test_mm_fp4.py b/tests/gemm/test_mm_fp4.py index 70eace91b1..5a634e34ef 100644 --- a/tests/gemm/test_mm_fp4.py +++ b/tests/gemm/test_mm_fp4.py @@ -32,13 +32,13 @@ def _test_mm_fp4( if backend == "cute-dsl": if not use_128x4_sf_layout: pytest.skip("cute_dsl backend only supports 128x4 SF layout") - if compute_capability[0] not in [10, 12]: - pytest.skip("cute_dsl backend only supports SM100/SM103/SM120/SM121 GPUs.") + if compute_capability[0] not in [10]: + pytest.skip("cute_dsl backend only supports SM100/SM103 GPUs.") if backend == "b12x": if not use_128x4_sf_layout: pytest.skip("b12x backend only supports 128x4 SF layout") - if compute_capability[0] != 12 or compute_capability[1] != 0: - pytest.skip("b12x backend only supports SM120 GPUs.") + if compute_capability[0] != 12: + pytest.skip("b12x backend only supports SM120/SM121 GPUs.") if not use_nvfp4: pytest.skip("b12x backend only supports NVFP4 (sf_vec_size=16).") if torch.version.cuda and int(torch.version.cuda.split(".")[0]) < 13: diff --git a/tests/moe/test_b12x_fused_moe.py b/tests/moe/test_b12x_fused_moe.py index 087426f25e..8973ec0ec0 100644 --- a/tests/moe/test_b12x_fused_moe.py +++ b/tests/moe/test_b12x_fused_moe.py @@ -65,14 +65,6 @@ def _cuda_13_or_newer(): return False -def _is_sm121(): - """Check whether the current device reports compute capability SM121.""" - if not torch.cuda.is_available(): - return False - props = torch.cuda.get_device_properties(0) - return props.major == 12 and props.minor == 1 - - # Skip decorators cute_dsl_available = pytest.mark.skipif( not is_cute_dsl_available(), reason="CuteDSL not available" @@ -85,10 +77,6 @@ def _is_sm121(): not _cuda_13_or_newer(), reason="b12x fused MoE requires CUDA 13 or later", ) -not_sm121 = pytest.mark.skipif( - _is_sm121(), - reason="b12x fused MoE is not supported on SM121", -) # ============================================================================= @@ -532,7 +520,6 @@ def create_relu2_moe_tensors( @cute_dsl_available @sm120_required @cuda_13_required -@not_sm121 class TestB12xFunctional: """Tests for the functional API: b12x_fused_moe.""" @@ -613,7 +600,6 @@ def test_numerical_accuracy( @cute_dsl_available @sm120_required @cuda_13_required -@not_sm121 class TestB12xWrapper: """Tests for the wrapper API: B12xMoEWrapper.""" @@ -795,7 +781,6 @@ def test_wrapper_cuda_graph(self, num_tokens: int, num_experts: int): @cute_dsl_available @sm120_required @cuda_13_required -@not_sm121 class TestB12xApiConsistency: """Tests verifying consistency between b12x functional and wrapper APIs.""" @@ -874,7 +859,6 @@ def test_functional_vs_wrapper_output(self): @cute_dsl_available @sm120_required @cuda_13_required -@not_sm121 class TestMicroKernel: """Tests for the micro kernel path (routed_rows <= 20-40). @@ -1076,7 +1060,6 @@ def test_micro_single_token_unique_path(self): @cute_dsl_available @sm120_required @cuda_13_required -@not_sm121 class TestRelu2Activation: """Tests for ReLU2 activation (non-gated, Nemotron-Super).""" From 4fa728cbbc01dac21d58ea1057b6f0409ccdee4b Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 18:17:11 +0000 Subject: [PATCH 06/11] Ensure setuptools>=77 and pip>=24 in Docker image The base Docker image may ship older setuptools that can't properly build the package. This matches the build-system requirement already declared in pyproject.toml. Co-authored-by: Cursor --- docker/install/install_python_packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/install/install_python_packages.sh b/docker/install/install_python_packages.sh index 6c980701b6..a76c55a41c 100644 --- a/docker/install/install_python_packages.sh +++ b/docker/install/install_python_packages.sh @@ -19,6 +19,8 @@ set -e set -u +pip3 install --upgrade "setuptools>=77" "pip>=24" + # Accept CUDA version as parameter (e.g., cu126, cu128, cu129) CUDA_VERSION=${1:-cu128} From b379ce36cb5ac5f3b8c64918ce77f719e8acec62 Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 18:26:05 +0000 Subject: [PATCH 07/11] Cap setuptools<82 to fix editable install with torch 2.11 setuptools 82.x has a regression with dynamic version fields and custom build backends, and torch 2.11.0+cu129 requires setuptools<82. Pin to >=77,<82 in both build-system requires and Docker setup. Co-authored-by: Cursor --- docker/install/install_python_packages.sh | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/install/install_python_packages.sh b/docker/install/install_python_packages.sh index a76c55a41c..be16a4e5eb 100644 --- a/docker/install/install_python_packages.sh +++ b/docker/install/install_python_packages.sh @@ -19,7 +19,7 @@ set -e set -u -pip3 install --upgrade "setuptools>=77" "pip>=24" +pip3 install --upgrade "setuptools>=77,<82" "pip>=24" # Accept CUDA version as parameter (e.g., cu126, cu128, cu129) CUDA_VERSION=${1:-cu128} diff --git a/pyproject.toml b/pyproject.toml index 19ae29c3c2..84010ea505 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ cu13 = ["nvidia-cutlass-dsl[cu13]>=4.5.0"] flashinfer = "flashinfer.__main__:cli" [build-system] -requires = ["setuptools>=77", "packaging>=24", "apache-tvm-ffi>=0.1.6,!=0.1.8,!=0.1.8.post0,<0.2"] +requires = ["setuptools>=77,<82", "packaging>=24", "apache-tvm-ffi>=0.1.6,!=0.1.8,!=0.1.8.post0,<0.2"] build-backend = "build_backend" backend-path = ["."] From b0edb4c1de7f31e79a8bbce4191a9aedfee20e13 Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 18:33:01 +0000 Subject: [PATCH 08/11] Add dynamic = ["version", "dependencies"] to [project] for PEP 621 compliance setuptools 82.x strictly enforces that dynamically-provided fields must be declared in the [project] dynamic list. Without this, editable installs fail with "project must contain ['version'] properties". This is the proper fix; the setuptools<82 cap is no longer needed. Co-authored-by: Cursor --- docker/install/install_python_packages.sh | 2 +- pyproject.toml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/install/install_python_packages.sh b/docker/install/install_python_packages.sh index be16a4e5eb..a76c55a41c 100644 --- a/docker/install/install_python_packages.sh +++ b/docker/install/install_python_packages.sh @@ -19,7 +19,7 @@ set -e set -u -pip3 install --upgrade "setuptools>=77,<82" "pip>=24" +pip3 install --upgrade "setuptools>=77" "pip>=24" # Accept CUDA version as parameter (e.g., cu126, cu128, cu129) CUDA_VERSION=${1:-cu128} diff --git a/pyproject.toml b/pyproject.toml index 84010ea505..9ea503dbd5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ [project] name = "flashinfer-python" +dynamic = ["version", "dependencies"] description = "FlashInfer: Kernel Library for LLM Serving" requires-python = ">=3.10,<4.0" authors = [{ name = "FlashInfer team" }] @@ -30,7 +31,7 @@ cu13 = ["nvidia-cutlass-dsl[cu13]>=4.5.0"] flashinfer = "flashinfer.__main__:cli" [build-system] -requires = ["setuptools>=77,<82", "packaging>=24", "apache-tvm-ffi>=0.1.6,!=0.1.8,!=0.1.8.post0,<0.2"] +requires = ["setuptools>=77", "packaging>=24", "apache-tvm-ffi>=0.1.6,!=0.1.8,!=0.1.8.post0,<0.2"] build-backend = "build_backend" backend-path = ["."] From 992f88d498dc8c4e7a0bdb2826c501def3e24d9d Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Wed, 6 May 2026 23:58:29 +0000 Subject: [PATCH 09/11] fix? --- requirements.txt | 1 - scripts/test_utils.sh | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c7534c8108..7e53dbeeb4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ einops ninja numpy nvidia-cudnn-frontend>=1.13.0 -nvidia-cutlass-dsl>=4.5.0 nvidia-ml-py packaging>=24.2 requests diff --git a/scripts/test_utils.sh b/scripts/test_utils.sh index d6a6d2c866..6c2424cc95 100755 --- a/scripts/test_utils.sh +++ b/scripts/test_utils.sh @@ -168,6 +168,14 @@ install_and_verify() { # Sync dependencies from the branch's requirements.txt pip install -r requirements.txt + # Install nvidia-cutlass-dsl with the correct CUDA extra to avoid + # version skew between libs-base and libs-cu13. + if [[ "${CUDA_VERSION}" == *"cu13"* ]] || [[ "${CUDA_VERSION}" == "13."* ]]; then + pip install --upgrade "nvidia-cutlass-dsl[cu13]>=4.5.0" + else + pip install --upgrade "nvidia-cutlass-dsl>=4.5.0" + fi + # Install local python sources pip install -e . -v --no-deps echo "" From 43495f438b87e9bd3357c8abb24c0c05240572a6 Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Thu, 7 May 2026 00:03:05 +0000 Subject: [PATCH 10/11] fix? --- requirements.txt | 1 + scripts/test_utils.sh | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 7e53dbeeb4..c7534c8108 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ einops ninja numpy nvidia-cudnn-frontend>=1.13.0 +nvidia-cutlass-dsl>=4.5.0 nvidia-ml-py packaging>=24.2 requests diff --git a/scripts/test_utils.sh b/scripts/test_utils.sh index 6c2424cc95..3f169f8ee1 100755 --- a/scripts/test_utils.sh +++ b/scripts/test_utils.sh @@ -170,10 +170,8 @@ install_and_verify() { # Install nvidia-cutlass-dsl with the correct CUDA extra to avoid # version skew between libs-base and libs-cu13. - if [[ "${CUDA_VERSION}" == *"cu13"* ]] || [[ "${CUDA_VERSION}" == "13."* ]]; then + if [[ "${CUDA_VERSION}" == *"cu13"* ]]; then pip install --upgrade "nvidia-cutlass-dsl[cu13]>=4.5.0" - else - pip install --upgrade "nvidia-cutlass-dsl>=4.5.0" fi # Install local python sources From 7ed00e576e9ce9252f3e8f5e4fedd6875e1acf2f Mon Sep 17 00:00:00 2001 From: kahyunnam Date: Thu, 7 May 2026 05:43:33 +0000 Subject: [PATCH 11/11] Fix b12x MoE runtime call for cutlass-dsl 4.5 Constexpr change MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cutlass-dsl 4.5.0 no longer exposes Constexpr parameters (like max_active_clusters) in the compiled kernel's runtime interface — they are baked in at cute.compile() time. Remove the trailing mac argument from static/micro and dynamic kernel launch calls. Co-authored-by: Cursor --- .../fused_moe/cute_dsl/blackwell_sm12x/moe_dispatch.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/flashinfer/fused_moe/cute_dsl/blackwell_sm12x/moe_dispatch.py b/flashinfer/fused_moe/cute_dsl/blackwell_sm12x/moe_dispatch.py index 2e83a78a6d..be56008568 100644 --- a/flashinfer/fused_moe/cute_dsl/blackwell_sm12x/moe_dispatch.py +++ b/flashinfer/fused_moe/cute_dsl/blackwell_sm12x/moe_dispatch.py @@ -913,7 +913,8 @@ def launch_sm120_static_moe( launch_ids = flat_ids # With TVM-FFI env stream, the stream is managed automatically. - # max_active_clusters is still a required positional arg for TVM-FFI. + # max_active_clusters (Constexpr) is baked in at cute.compile() time + # and must NOT be passed at runtime (cutlass-dsl >= 4.5). # Pointer arguments must be passed as raw ints (data_ptr()) at runtime. compiled( a, @@ -940,7 +941,6 @@ def launch_sm120_static_moe( scatter_output, workspace.token_map, workspace.token_weights, - mac, ) return scatter_output @@ -1520,6 +1520,8 @@ def launch_sm120_dynamic_moe( # Dynamic kernel: runtime-shaped args are DataPointer (pass data_ptr()), # fixed-shape args are Tensor (pass torch tensor directly). + # max_active_clusters (Constexpr) is baked in at cute.compile() time + # and must NOT be passed at runtime (cutlass-dsl >= 4.5). compiled( a.data_ptr(), flat_ids.data_ptr(), @@ -1561,7 +1563,6 @@ def launch_sm120_dynamic_moe( workspace.physical_tiles_capacity * _LEVEL_TILE_M, workspace.task_capacity, workspace.physical_tiles_capacity, - mac, ) return scatter_output