[https://nvbugs/6640776][fix] Bump CUTLASS DSL to 4.6.2 to unblock FA4 split-KV

constraints.txt

@@ -1,12 +1,12 @@
 # setup.py appends these constraints to wheel install_requires, so every entry
 # must also be a valid TensorRT-LLM runtime dependency outside the base image.
 # Keep the base-image and wheel CuTe DSL stack aligned with requirements.txt.
-nvidia-cutlass-dsl==4.6.1
+nvidia-cutlass-dsl==4.6.2
 # Keep the CuTe DSL and FlashAttention 4 runtime stack aligned with requirements.txt.
 apache-tvm-ffi==0.1.13.post2
-quack-kernels==0.6.2
+quack-kernels==0.6.4
 torch-c-dlpack-ext==0.1.3
-# CUTLASS DSL 4.6.1 libraries require protobuf 6.x.
+# The CUTLASS DSL libraries require protobuf 6.x.
 protobuf>=6.30.2,<7
 # The `nvidia-cutlass-dsl` package does not pin numpy at all, which can be problematic in certain CI
 # stages.

docker/Dockerfile.multi

@@ -72,7 +72,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
         nvidia-cutlass-dsl-libs-core nvidia-cutlass-dsl-libs-cu12 nvidia-cutlass-dsl-libs-cu13 numpy wandb || true && \
     # Remove any leftover namespace dirs or dist-info that pip missed
     rm -rf $(python3 -c "import site; print(site.getsitepackages()[0])")/nvidia_cutlass_dsl* && \
-    pip3 install -r /tmp/constraints.txt "nvidia-cutlass-dsl[cu13]==4.6.1" && \
+    pip3 install -r /tmp/constraints.txt "nvidia-cutlass-dsl[cu13]==4.6.2" && \
     rm /tmp/constraints.txt
 
 # Install UCX, NIXL, etcd

requirements.txt

@@ -78,15 +78,17 @@ xdsl>=0.59.0 # Optional: required for MLIR-based elementwise fusion in AutoDeplo
 tiktoken
 blobfile
 openai-harmony==0.0.4
-nvidia-cutlass-dsl[cu13]==4.6.1; python_version >= "3.10" # required by Blackwell CuTe DSL kernels
+# 4.6.1 rejects FlashAttention 4's sm100 split-KV kernel (TYPE_UNSTABLE_JOIN).
+nvidia-cutlass-dsl[cu13]==4.6.2; python_version >= "3.10" # required by Blackwell CuTe DSL kernels
 nvidia-matmul-heuristics==0.1.0.27; python_version >= "3.10" # analytic GEMM heuristics for CuTe DSL autotuner tactic pruning
-quack-kernels==0.6.2; python_version >= "3.10" # required for MinimaxM3 MSA and FlashAttention 4; pins CUTLASS DSL 4.6.1
+# quack-kernels pins the CUTLASS DSL by equality, so bump the two together.
+quack-kernels==0.6.4; python_version >= "3.10" # required for MinimaxM3 MSA and FlashAttention 4
 jinja2 # required for MinimaxM3 MSA
 plotly
 numexpr
 partial_json_parser
 mcp<2.0.0 # 2.0.0 removed mcp.server.fastmcp; scaffolding still uses the 1.x APIs
-apache-tvm-ffi==0.1.13.post2 # required by CUTLASS DSL 4.6.1's TVM-FFI provider
+apache-tvm-ffi==0.1.13.post2 # required by the CUTLASS DSL TVM-FFI provider
 torch-c-dlpack-ext==0.1.3 # used for reduce nvidia-cutlass-dsl host overhead, optional package for improved torch tensor calling perf
 flash-attn-4==4.0.0b19
 mistral-common>=1.10.0

security_scanning/pyproject.toml

@@ -70,9 +70,9 @@ dependencies = [
     "tiktoken (>=0.14.0,<0.15.0)",
     "blobfile (>=3.3.0,<4.0.0)",
     "openai-harmony (==0.0.4)",
-    "nvidia-cutlass-dsl[cu13] (==4.6.1)",
+    "nvidia-cutlass-dsl[cu13] (==4.6.2)",
     "nvidia-matmul-heuristics (==0.1.0.27)",
-    "quack-kernels (==0.6.2)",
+    "quack-kernels (==0.6.4)",
     "jinja2 (>=3.1.6,<4.0.0)",
     "plotly (>=7.0.0,<8.0.0)",
     "partial-json-parser (>=0.2.1.1.post7,<0.3.0.0)",

tensorrt_llm/_torch/modules/fused_moe/fused_moe_cute_dsl.py

@@ -25,12 +25,7 @@
 
 from ...autotuner import (AutoTuner, ConstraintSpec, DynamicTensorSpec,
                           OptimizationProfile, TunableRunner, TuningConfig)
-from ...custom_ops.cute_dsl_custom_ops import (
-    GroupedGemmInputsHelper,
-    Sm100BlockScaledContiguousGatherGroupedGemmActFusionRunner,
-    Sm100BlockScaledContiguousGroupedGemmFinalizeFusionRunner,
-    Sm100BlockScaledContiguousGroupedGemmRunner,
-    Sm100BlockScaledContiguousGroupedGemmSwigluFusionRunner)
+from ...custom_ops.cute_dsl_custom_ops import GroupedGemmInputsHelper
 from ...model_config import ModelConfig
 from ...utils import (ActivationType, AuxStreamType, EventType,
                       Fp4QuantizedTensor,
@@ -326,6 +321,19 @@ def runner_tactic_comb_checker(
         if tile_size is None:
             return True
 
+        # Imported here rather than at module scope because these classes are
+        # defined inside cute_dsl_custom_ops' ``if IS_CUTLASS_DSL_AVAILABLE:``
+        # block: a module-scope import makes this file -- and therefore
+        # create_moe, modeling_utils and every model -- unimportable without the
+        # optional cutlass DSL. Reaching this line means a CuteDSL runner is
+        # already being autotuned, so the DSL is installed. Same pattern as
+        # attention_backend/fmha/cute_dsl_mla.py and sparse/dsa/metadata.py.
+        from ...custom_ops.cute_dsl_custom_ops import (
+            Sm100BlockScaledContiguousGatherGroupedGemmActFusionRunner,
+            Sm100BlockScaledContiguousGroupedGemmFinalizeFusionRunner,
+            Sm100BlockScaledContiguousGroupedGemmRunner,
+            Sm100BlockScaledContiguousGroupedGemmSwigluFusionRunner)
+
         for runner, tactic in comb:
             if isinstance(
                     runner,

tensorrt_llm/_torch/modules/fused_moe/mega_moe/mega_moe_cute_dsl.py

@@ -137,10 +137,10 @@ def is_megamoe_cute_dsl_runtime_available() -> Tuple[bool, Optional[str]]:
     ABI also requires ``cutlass.torch.from_dlpack``, ``cutlass._mlir``
     APIs used by ``sym_buffer.py``, the ``cute_nvgpu`` MMA atoms used
     by ``kernel_fc12.py``, and the async-copy helpers used by
-    ``dispatch_kernel.py``. PR
-    https://github.com/NVIDIA/TensorRT-LLM/pull/14354 pins
-    ``nvidia-cutlass-dsl[cu13]==4.6.1``; version 4.5.0 was the first release
-    that shipped all of them, and older wheels return ``(False, reason)``.
+    ``dispatch_kernel.py``. ``nvidia-cutlass-dsl[cu13]`` 4.5.0 was the first
+    release that shipped all of them (see PR
+    https://github.com/NVIDIA/TensorRT-LLM/pull/14354), and older wheels
+    return ``(False, reason)``.
 
     Returns ``(True, None)`` on success or ``(False, reason)`` with an
     actionable message. The result is cached for the process lifetime.

tensorrt_llm/_torch/visual_gen/attention_backend/_flash_attn_cute.py

@@ -0,0 +1,45 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Resolution of optional FlashAttention 4 CuTe symbols.
+
+Shared by the backends that consume ``flash_attn.cute.interface`` so the
+optional-dependency contract is defined once instead of per import site.
+"""
+
+import importlib
+from typing import Any, Callable, Optional
+
+
+def resolve_flash_attn_cute_symbol(
+    name: str,
+) -> tuple[Optional[Callable[..., Any]], Optional[BaseException]]:
+    """Return ``(symbol, None)`` if FlashAttention 4 provides ``name``, else ``(None, error)``.
+
+    Callers keep the returned error and raise it at first use, so an unusable FA4
+    degrades to a ``None`` sentinel instead of breaking module import.
+
+    Every failure mode is treated as "FA4 unusable" rather than enumerating
+    exception types: a *broken* install -- present, but built against a different
+    CuTe DSL -- fails while executing ``flash_attn``'s module body and surfaces as
+    whatever that line happens to raise (e.g. ``AttributeError`` for a DSL symbol
+    that has since been removed), not as ``ImportError``. This package's
+    ``__init__`` imports its backends unconditionally, so an escaping exception
+    would take down every importer of ``visual_gen`` -- including ``VisualGen``
+    itself, whose model packages pull in ``attention_backend.parallel``.
+    """
+    try:
+        return getattr(importlib.import_module("flash_attn.cute.interface"), name), None
+    except Exception as e:  # noqa: BLE001 - any failure means FA4 is unusable here
+        return None, e

tensorrt_llm/_torch/visual_gen/attention_backend/flash_attn4.py

@@ -25,14 +25,10 @@
 import torch
 
 from ...attention_backend.interface import PredefinedAttentionMask
+from ._flash_attn_cute import resolve_flash_attn_cute_symbol
 from .interface import AttentionBackend, AttentionTensorLayout
 
-_flash_attn_fwd_import_error = None
-try:
-    from flash_attn.cute.interface import _flash_attn_fwd
-except (ImportError, OSError) as e:
-    _flash_attn_fwd = None
-    _flash_attn_fwd_import_error = e
+_flash_attn_fwd, _flash_attn_fwd_import_error = resolve_flash_attn_cute_symbol("_flash_attn_fwd")
 
 
 class FlashAttn4Attention(AttentionBackend):

tensorrt_llm/_torch/visual_gen/attention_backend/parallel.py

@@ -32,14 +32,12 @@
 from tensorrt_llm._torch.distributed import all_to_all_4d, all_to_all_5d
 
 from ...attention_backend.interface import PredefinedAttentionMask
+from ._flash_attn_cute import resolve_flash_attn_cute_symbol
 from .interface import AttentionBackend, AttentionTensorLayout
 
-_flash_attn_combine_import_error = None
-try:
-    from flash_attn.cute.interface import flash_attn_combine as _flash_attn_combine
-except (ImportError, OSError) as e:
-    _flash_attn_combine = None
-    _flash_attn_combine_import_error = e
+_flash_attn_combine, _flash_attn_combine_import_error = resolve_flash_attn_cute_symbol(
+    "flash_attn_combine"
+)
 
 
 def post_permute_5d_to_4d(out_5d, P):

tests/integration/test_lists/waives.txt

@@ -113,7 +113,6 @@ examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[f
 examples/visual_gen/test_visual_gen_flux.py::test_flux_accuracy_against_golden[flux2-nvfp4] SKIP (https://nvbugs/6572800)
 examples/visual_gen/test_visual_gen_glm.py::test_glm_image_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6644450)
 examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_cuda_graph_trtllm_backend SKIP (https://nvbugs/6668775)
-examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_example SKIP (https://nvbugs/6641122)
 examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[cuda-graph] SKIP (https://nvbugs/6655986)
 examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_feature_accuracy_against_golden[nvfp4] SKIP (https://nvbugs/6572800)
 examples/visual_gen/test_visual_gen_ltx2.py::test_ltx2_lpips_against_golden SKIP (https://nvbugs/6655986)