From 6659ac20f758903be0fd1804860d22fb495b9969 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 24 Sep 2026 22:59:04 +0000 Subject: [PATCH 01/10] [Perf][DSv4.1] Fuse small-batch WO-A with inverse RoPE and MXFP8 quant on SM100 For 1-24 tokens, replace the Triton inverse-RoPE FP8 quant, DeepGEMM fp8_einsum and FlashInfer MXFP8 quantize ahead of wo_b with one CuTe-DSL kernel that emits wo_b's pre-quantized MXFP8 input. The kernel follows the VllmCuTeDSLJitKernel warmup contract and is registered by the FlashMLA and FlashInfer SM100 attention backends. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Juntian Liu --- .../kernels/test_fused_inv_rope_fp8_quant.py | 73 ++++ vllm/models/deepseek_v41/attention.py | 4 +- .../deepseek_v41/nvidia/flashinfer_sparse.py | 16 +- vllm/models/deepseek_v41/nvidia/flashmla.py | 13 +- .../deepseek_v41/nvidia/ops/fused_wo_a.py | 366 ++++++++++++++++++ vllm/models/deepseek_v41/nvidia/ops/o_proj.py | 150 +++++++ 6 files changed, 609 insertions(+), 13 deletions(-) create mode 100644 vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py create mode 100644 vllm/models/deepseek_v41/nvidia/ops/o_proj.py diff --git a/tests/kernels/test_fused_inv_rope_fp8_quant.py b/tests/kernels/test_fused_inv_rope_fp8_quant.py index 03d5b7921fa2..ab8b31ef2a05 100644 --- a/tests/kernels/test_fused_inv_rope_fp8_quant.py +++ b/tests/kernels/test_fused_inv_rope_fp8_quant.py @@ -33,6 +33,79 @@ EPS = 1e-10 +@pytest.mark.parametrize("tokens", [1, 3, 6, 8, 12, 16, 18, 32]) +@pytest.mark.parametrize("padded,zero", [(False, False), (True, False), (True, True)]) +def test_dsv41_fused_wo_a_matches_deepgemm(tokens, padded, zero): + """Preserve per-32 WO-A math and WO-B scales, including graph replay.""" + from vllm.platforms import current_platform + + if not current_platform.is_device_capability_family(100): + pytest.skip("Requires SM100/SM103") + pytest.importorskip("cutlass") + from flashinfer import mxfp8_quantize + + from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + deepgemm_post_process_fp8_weight_block, + ) + from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL + from vllm.utils.deep_gemm import fp8_einsum + + torch.manual_seed(42) + x = torch.randn( + tokens, 64 if padded else 16, 512, device="cuda", dtype=torch.bfloat16 + )[:, :16] + x.mul_(torch.exp2(torch.arange(16, device="cuda") - 8)[None, :, None]) + if zero: + x.zero_() + positions = torch.randint(0, 128, (tokens,), device="cuda") + rope = make_cos_sin_cache(128, device="cuda") + w = torch.randn(2048, 4096, device="cuda", dtype=torch.bfloat16) + wb = w.float().view(2048, 128, 32) + scale = torch.exp2(torch.ceil(torch.log2(wb.abs().amax(-1) / 448))) + wq = (wb / scale[..., None]).to(FP8_DTYPE).view(2048, 4096) + wq, ws = deepgemm_post_process_fp8_weight_block(wq, scale, (1, 32), False, True, 2) + inputs = dict(x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws) + _FUSED_WO_A_KERNEL(**inputs) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + q, sf = _FUSED_WO_A_KERNEL(**inputs) + for _ in range(2): + positions.add_(1).remainder_(128) + graph.replay() + a, sa = fused_inv_rope_fp8_quant( + x, + positions, + rope, + n_groups=2, + heads_per_group=8, + nope_dim=448, + rope_dim=64, + quant_group_size=32, + tma_aligned_scales=True, + ) + y = torch.empty(tokens, 2, 1024, device="cuda", dtype=torch.bfloat16) + fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32)) + ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl") + + def dequant(data, scales): + scales = scales.reshape(16, 32, 4, 4).permute(1, 2, 0, 3) + scales = scales.reshape(128, 64)[:tokens].float() + return data.float().reshape(tokens, 64, 32) * torch.exp2( + scales[..., None] - 127 + ) + + actual, expected = dequant(q, sf), dequant(ref, ref_sf) + assert torch.isfinite(actual).all() + if zero: + assert torch.count_nonzero(q.float()) == 0 + else: + error = (actual - expected).norm() / expected.norm() + assert error < 0.003, error.item() + rows = torch.arange(512, device="cuda") + rows = rows // 16 + (rows % 16 // 4) * 32 + assert torch.count_nonzero(sf.view(16, 512)[:, rows >= tokens]) == 0 + + # ========================================================================= # Helpers # ========================================================================= diff --git a/vllm/models/deepseek_v41/attention.py b/vllm/models/deepseek_v41/attention.py index d0e69fb35871..6d961f0055de 100644 --- a/vllm/models/deepseek_v41/attention.py +++ b/vllm/models/deepseek_v41/attention.py @@ -697,7 +697,7 @@ def bind_gemm_rs(self) -> None: else: gemm_rs.warn_incompatible_projection() - def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor: + def _wo_b_proj(self, z: torch.Tensor | QuantizedActivation) -> torch.Tensor: """Apply ``wo_b``; with GEMM-RS bound, also reduce-scatter the result. Every ``_o_proj`` implementation projects through this so the decoder @@ -707,7 +707,7 @@ def _wo_b_proj(self, z: torch.Tensor) -> torch.Tensor: """ if self.gemm_rs is None: return self.wo_b(z) - if self.gemm_rs.should_run(z): + if isinstance(z, torch.Tensor) and self.gemm_rs.should_run(z): return self.gemm_rs.apply(z, self.wo_b) # Small batches stay on the unfused path, which is faster there. return sp_reduce_scatter(self.wo_b(z)) diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index 423849dcf1eb..358dcc78f79e 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -9,15 +9,16 @@ from vllm.config import VllmConfig from vllm.config.cache import CacheDType from vllm.forward_context import get_forward_context -from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( - compute_fp8_einsum_recipe, - deep_gemm_fp8_o_proj, -) +from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.common.ops import ( build_flashinfer_mixed_sparse_indices, compute_global_topk_indices_and_lens, ) +from vllm.models.deepseek_v41.nvidia.ops.o_proj import ( + dsv41_o_proj, + register_dsv41_o_proj_warmup, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLAMetadata, DeepseekV4SparseMLABackend, @@ -243,12 +244,13 @@ def get_padded_num_q_heads(cls, num_heads: int) -> int: def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( + return dsv41_o_proj( o, positions, self.rotary_emb.cos_sin_cache, self.wo_a, self._wo_b_proj, + wo_b_layer=self.wo_b, n_groups=self.n_local_groups, heads_per_group=self.n_local_heads // self.n_local_groups, nope_dim=self.nope_head_dim, @@ -263,6 +265,7 @@ def __init__(self, *args, **kwargs) -> None: self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe( self._o_proj_block_size ) + register_dsv41_o_proj_warmup(self) # Per-tensor FP8 scale buffers + precomputed scalar BMM scales. Only the # per-tensor FP8 cache path consumes these; bf16 reads ``self.scale``. if self.kv_cache_torch_dtype != torch.float8_e4m3fn: @@ -624,12 +627,13 @@ def get_padded_num_q_heads(cls, num_heads: int) -> int: def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( + return dsv41_o_proj( o, positions, self.rotary_emb.cos_sin_cache, self.wo_a, self._wo_b_proj, + wo_b_layer=self.wo_b, n_groups=self.n_local_groups, heads_per_group=self.n_local_heads // self.n_local_groups, nope_dim=self.nope_head_dim, diff --git a/vllm/models/deepseek_v41/nvidia/flashmla.py b/vllm/models/deepseek_v41/nvidia/flashmla.py index 96dde36c61d1..0b7ecb5f12e1 100644 --- a/vllm/models/deepseek_v41/nvidia/flashmla.py +++ b/vllm/models/deepseek_v41/nvidia/flashmla.py @@ -6,16 +6,17 @@ import torch from vllm.forward_context import get_forward_context -from vllm.models.deepseek_v4.nvidia.ops.o_proj import ( - compute_fp8_einsum_recipe, - deep_gemm_fp8_o_proj, -) +from vllm.models.deepseek_v4.nvidia.ops.o_proj import compute_fp8_einsum_recipe from vllm.models.deepseek_v41.attention import DeepseekV4Attention from vllm.models.deepseek_v41.common.ops import ( combine_topk_swa_indices, compute_global_topk_indices_and_lens, dequantize_and_gather_k_cache, ) +from vllm.models.deepseek_v41.nvidia.ops.o_proj import ( + dsv41_o_proj, + register_dsv41_o_proj_warmup, +) from vllm.models.deepseek_v41.sparse_mla import ( DeepseekV4FlashMLABackend, DeepseekV4FlashMLAMetadata, @@ -57,15 +58,17 @@ def __init__(self, *args, **kwargs) -> None: self._einsum_recipe, self._tma_aligned_scales = compute_fp8_einsum_recipe( self._o_proj_block_size ) + register_dsv41_o_proj_warmup(self) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: o = attn_out[:, : self.n_local_heads, :] - return deep_gemm_fp8_o_proj( + return dsv41_o_proj( o, positions, self.rotary_emb.cos_sin_cache, self.wo_a, self._wo_b_proj, + wo_b_layer=self.wo_b, n_groups=self.n_local_groups, heads_per_group=self.n_local_heads // self.n_local_groups, nope_dim=self.nope_head_dim, diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py new file mode 100644 index 000000000000..19e42585ff11 --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -0,0 +1,366 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DSV4.1 small-batch MXFP8 WO-A with inverse RoPE and quantization.""" + +from dataclasses import dataclass +from typing import Any + +import cutlass +import cutlass.cute as cute +import cutlass.utils as utils +import torch +from cuda.bindings.driver import CUstream +from cutlass import ( + BFloat16, + Float8E4M3FN, + Float32, + Int32, + Int64, + Uint8, + Uint32, + Uint64, +) +from cutlass._mlir.dialects import llvm +from cutlass.cute.nvgpu import cpasync, tcgen05 +from cutlass.cute.runtime import make_fake_tensor +from cutlass.cutlass_dsl import dsl_user_op + +from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy +from vllm.cute_utils.cvt import bf16x2_to_fp32x2, fp32x4_to_fp8x4 +from vllm.model_executor.warmup.jit_warmup import kernel_launcher +from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( + CuTeDSLLaunchSpec, + VllmCuTeDSLJitKernel, +) + + +@dsl_user_op +def _ue8m0_rp(value, *, loc=None, ip=None): + result = llvm.inline_asm( + Uint32.mlir_type, + [value.ir_value(loc=loc, ip=ip)], + "{ .reg .b16 sf; cvt.rp.satfinite.ue8m0x2.f32 sf, $1, $1; " + "cvt.u32.u16 $0, sf; and.b32 $0, $0, 255; }", + "=r,f", + has_side_effects=False, + loc=loc, + ip=ip, + ) + return Uint32(result) + + +@cute.jit +def _scale(amax: Float32): + exponent = _ue8m0_rp(amax * Float32(1.0 / 448.0)) + inv = recast_val((Uint32(254) - exponent) << 23, Float32) + if exponent == 0: + inv = Float32(0) + return exponent, inv + + +class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]): + @dataclass(frozen=True) + class CompileKey: + tokens: int + x_stride: int + + @staticmethod + def kernel(compile_key: CompileKey) -> Any: + tokens = compile_key.tokens + acc_cols = 1 << (tokens - 1).bit_length() + tile_m = max(8, acc_cols) + sf_col = max(16, tile_m) + tmem_cols = sf_col * 2 + threads = min(acc_cols * 128, 512) + + @cute.kernel + def device_kernel( + x: cute.Tensor, + positions: cute.Tensor, + rope: cute.Tensor, + w: cpasync.TmaInfo, + ws: cute.Tensor, + q: cute.Tensor, + qs: cute.Tensor, + ): + tid, _, _ = cute.arch.thread_idx() + bid, _, _ = cute.arch.block_idx() + warp = cute.arch.make_warp_uniform(tid // 32) + lane = tid % 32 + split = bid % 8 + tile = bid // 8 + group = tile // 8 + smem = utils.SmemAllocator() + sw = smem.allocate_tensor( + Float8E4M3FN, + w.smem_layout.outer, + byte_alignment=128, + swizzle=w.smem_layout.inner, + ) + sx = smem.allocate_tensor( + Float8E4M3FN, + cute.make_layout((tile_m, 128, 4), stride=(128, 1, tile_m * 128)), + byte_alignment=1024, + ) + ssw = smem.allocate_tensor(Int32, cute.make_layout((128, 4)), 128) + ssx = smem.allocate_tensor(Int32, cute.make_layout((128, 4)), 128) + partial = smem.allocate_tensor( + Float32, cute.make_layout((128, (tokens + 7) // 8, 8)), 128 + ) + loaded = smem.allocate_array(Int64, 4) + done = smem.allocate_array(Int64, 1) + reduced = smem.allocate_array(Int64, 1) + taddr = smem.allocate(Int32, 4) + if tid == 0: + for stage in cutlass.range_constexpr(4): + cute.arch.mbarrier_init(loaded + stage, 1) + cute.arch.mbarrier_init(done, 1) + cute.arch.mbarrier_init(reduced, 1) + cute.arch.mbarrier_init_fence() + cute.arch.sync_threads() + cute.arch.cluster_arrive_relaxed() + if warp == 0: + tiles = cute.zipped_divide(w.tma_tensor, (128, 128)) + for stage in cutlass.range_constexpr(4): + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(loaded + stage, 16384) + simple_tma_copy( + w.atom, + tiles[None, (tile, split * 4 + stage)], + sw[None, None, stage], + loaded + stage, + ) + elif warp == 1: + cute.arch.alloc_tmem(tmem_cols, taddr) + cute.arch.relinquish_tmem_alloc_permit() + if tid < 128: + for stage in cutlass.range_constexpr(4): + ssw[(tid % 32) * 4 + tid // 32, stage] = ws[ + group, (tile % 8) * 128 + tid, split * 4 + stage + ] + ssx[tid, stage] = Int32(0) + sx32 = cute.recast_tensor(sx, Int32) + for row in cutlass.range_constexpr(tile_m // 4): + sx32[tid // 32 + row * 4, tid % 32, stage] = Int32(0) + cute.arch.sync_threads() + cute.arch.griddepcontrol_wait() + x64 = cute.recast_tensor(x, Uint64) + sx32 = cute.recast_tensor(sx, Uint32) + sfbytes = cute.recast_tensor(ssx, Uint8) + for iteration in cutlass.range_constexpr( + (tokens * 128 + threads - 1) // threads + ): + token = iteration * (threads // 128) + tid // 128 + qid = tid % 128 + k = qid * 4 + if cutlass.const_expr(tokens % (threads // 128) == 0) or token < tokens: + packed = x64[token, group * 8 + split, qid] + v0, v1 = bf16x2_to_fp32x2(Uint32(packed)) + v2, v3 = bf16x2_to_fp32x2(Uint32(packed >> 32)) + if k >= 448: + pos = positions[token] + c = Float32(rope[pos, (k - 448) // 2]) + s = Float32(rope[pos, (k - 448) // 2 + 32]) + v0, v1 = v0 * c + v1 * s, v1 * c - v0 * s + c = Float32(rope[pos, (k - 448) // 2 + 1]) + s = Float32(rope[pos, (k - 448) // 2 + 33]) + v2, v3 = v2 * c + v3 * s, v3 * c - v2 * s + amax = cute.arch.fmax( + cute.arch.fmax(cute.abs(v0), cute.abs(v1)), + cute.arch.fmax(cute.abs(v2), cute.abs(v3)), + ) + amax = cute.arch.warp_reduction_max(amax, threads_in_group=8) + exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10))) + # Recasting drops the pointer swizzle; address it explicitly. + sx32[token, (qid % 32) ^ ((token % 8) * 4), qid // 32] = ( + fp32x4_to_fp8x4(v0 * inv, v1 * inv, v2 * inv, v3 * inv) + ) + if lane % 8 == 0: + sfbytes[token * 16 + k % 128 // 32 + k // 128 * 512] = Uint8( + exponent + ) + cute.arch.sync_threads() + cute.arch.fence_proxy("async.shared", space="cta") + _tcgen05.fence_after_thread_sync() + if tid == 0: + cute.arch.griddepcontrol_launch_dependents() + base = cute.make_tensor(taddr, cute.make_layout(1))[0] + if warp == 0: + sdesc = _tcgen05.make_sdesc_128B_swizzle(0) + sfdesc = Uint64((8 << 32) | (1 << 46)) + idesc = _tcgen05.make_mxfp8_idesc(128, tile_m) + for stage in cutlass.range_constexpr(4): + cute.arch.mbarrier_wait(loaded + stage, 0) + adesc = sdesc | (sw[None, None, stage].iterator.toint() >> 4) + bdesc = sdesc | (sx[None, None, stage].iterator.toint() >> 4) + _tcgen05.cp( + base + sf_col, + sfdesc | (ssw[None, stage].iterator.toint() >> 4), + "32x128b", + "warpx4", + ) + _tcgen05.cp( + base + sf_col + 4, + sfdesc | (ssx[None, stage].iterator.toint() >> 4), + "32x128b", + "warpx4", + ) + for kk in cutlass.range_constexpr(4): + _tcgen05.mma_mxfp8( + base, + adesc + kk * 2, + bdesc + kk * 2, + idesc + ((kk << 4) | (kk << 29)), + base + sf_col, + base + sf_col + 4, + stage > 0 or kk > 0, + ) + _tcgen05.commit(done) + cute.arch.mbarrier_wait(done, 0) + _tcgen05.fence_after_thread_sync() + if tid == 0: + owned = (tokens + 7 - split) // 8 + mbarrier.arrive_expect_tx(reduced, owned * 128 * 8 * 4) + cute.arch.cluster_wait() + if tid < 128: + acc = cute.make_rmem_tensor(acc_cols, Float32) + if cutlass.const_expr(tokens == 1): + acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1) + else: + acc.store(_tcgen05.ld(warp * 32, base, "32x32b", acc_cols)) + _tcgen05.wait_ld() + for token in cutlass.range_constexpr(tokens): + ptr = cute.domain_offset((tid, token // 8, split), partial).iterator + cute.arch.store_async_dsmem( + ptr, recast_val(acc[token], Int32), reduced, token % 8 + ) + if split < tokens: + cute.arch.mbarrier_wait(reduced, 0) + for i in cutlass.range_constexpr((tokens + 7) // 8): + token = split + i * 8 + if ( + cutlass.const_expr(tokens <= 8 or tokens % 8 == 0) + or token < tokens + ): + value = Float32(0) + for peer in cutlass.range_constexpr(8): + value += partial[tid, i, peer] + value = Float32(BFloat16(value)) + amax = cute.arch.warp_reduction_max(cute.abs(value)) + exponent, inv = _scale(amax) + q[token, tile * 128 + tid] = Float8E4M3FN(value * inv) + if lane == 0: + qs[tile * 512 + token * 16 + warp] = Uint8(exponent) + # Every scale tile owns its padding, with no second memset kernel. + for i in cutlass.range_constexpr(4): + offset = i * 128 + tid + row = offset // 16 + (offset % 16 // 4) * 32 + if split == 0 and row >= tokens: + qs[tile * 512 + offset] = Uint8(0) + # Each receiver drained all writes into its own inbox. Only local + # TMEM readers must finish before deallocation; no peer reads SMEM. + cute.arch.sync_threads() + if warp == 0: + cute.arch.dealloc_tmem( + cute.make_ptr(Float32, base, cute.AddressSpace.tmem), tmem_cols + ) + + @cute.jit + def host_entrypoint( + x: cute.Tensor, + positions: cute.Tensor, + rope: cute.Tensor, + w: cute.Tensor, + ws: cute.Tensor, + q: cute.Tensor, + qs: cute.Tensor, + stream: CUstream, + ): + layout = cute.make_composed_layout( + cute.make_swizzle(3, 4, 3), + 0, + cute.make_layout((128, 128, 4), stride=(128, 1, 16384)), + ) + tma = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), + w, + layout, + (128, 128), + ) + device_kernel(x, positions, rope, tma, ws, q, qs).launch( + grid=(128, 1, 1), + block=(threads, 1, 1), + cluster=(8, 1, 1), + stream=stream, + use_pdl=True, + ) + + return host_entrypoint + + def dispatch( # type: ignore[override] + self, *, tokens: int, x_stride: int + ) -> CompileKey: + return self.CompileKey(tokens=tokens, x_stride=x_stride) + + def get_warmup_keys( + self, vllm_config: Any, *, max_tokens: int, x_stride: int + ) -> list[CompileKey]: + compilation_config = vllm_config.compilation_config + # Graphs pad batches up to a capture size; eager batches keep theirs. + padded = 0 + if compilation_config.cudagraph_mode.mixed_mode(): + padded = compilation_config.max_cudagraph_capture_size or 0 + captured = compilation_config.cudagraph_capture_sizes or [] + return self._trace_dispatch(self.dispatch)( + tokens=[ + *(size for size in captured if size <= min(padded, max_tokens)), + *range(padded + 1, max_tokens + 1), + ], + x_stride=x_stride, + ) + + def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: + tokens = compile_key.tokens + return ( + make_fake_tensor( + BFloat16, + (tokens, 16, 512), + (compile_key.x_stride, 512, 1), + assumed_align=16, + ), + make_fake_tensor(Int64, (tokens,), (1,)), + make_fake_tensor(Float32, (cute.sym_int(), 64), (64, 1)), + make_fake_tensor(Float8E4M3FN, (2048, 4096), (4096, 1), assumed_align=16), + make_fake_tensor(Int32, (2, 1024, 32), (32768, 1, 1024)), + make_fake_tensor(Float8E4M3FN, (tokens, 2048), (2048, 1)), + make_fake_tensor(Uint8, (8192,), (1,)), + ) + + @kernel_launcher + def __call__( + self, + *, + x: torch.Tensor, + positions: torch.Tensor, + rope: torch.Tensor, + weight: torch.Tensor, + weight_scale: torch.Tensor, + ) -> CuTeDSLLaunchSpec["FusedWoAKernel.CompileKey"]: + """Return FP8 WO-A output and FlashInfer F8_128x4 scale bytes.""" + tokens = x.shape[0] + q = torch.empty((tokens, 2048), device=x.device, dtype=torch.float8_e4m3fn) + scales = torch.empty(8192, device=x.device, dtype=torch.uint8) + launch_args = ( + x, + positions, + rope, + weight.view(2048, 4096), + weight_scale, + q, + scales, + ) + compile_key = self.dispatch(tokens=tokens, x_stride=x.stride(0)) + return compile_key, launch_args, (q, scales) + + +_FUSED_WO_A_KERNEL = FusedWoAKernel() diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py new file mode 100644 index 000000000000..279f4658181e --- /dev/null +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -0,0 +1,150 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""DSV4.1 output projection with a small-batch SM100 fusion.""" + +from collections.abc import Callable + +import torch +from torch import nn + +from vllm.config import get_current_vllm_config +from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( + FlashInferCutedslMxfp8LinearKernel, +) +from vllm.model_executor.layers.fusion.quant_activation import ( + QuantizedActivation, + get_input_quant_key, +) +from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic +from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj +from vllm.platforms import current_platform +from vllm.utils.flashinfer import has_flashinfer_cutedsl + +# GB200: gains hold through 24 tokens; larger graphs have marginal benefit. +_FUSED_WO_A_MAX_TOKENS = 24 + + +def _can_fuse_wo_a( + wo_b_layer: nn.Module, + n_groups: int, + heads_per_group: int, + nope_dim: int, + rope_dim: int, + o_lora_rank: int, + einsum_recipe: tuple[int, int, int], + tma_aligned_scales: bool, +) -> bool: + method = getattr(wo_b_layer, "scheme", getattr(wo_b_layer, "quant_method", None)) + return ( + (n_groups, heads_per_group, nope_dim, rope_dim, o_lora_rank) + == (2, 8, 448, 64, 1024) + and einsum_recipe == (1, 1, 32) + and tma_aligned_scales + and current_platform.is_device_capability_family(100) + and has_flashinfer_cutedsl() + and get_input_quant_key(wo_b_layer) == kMxfp8Dynamic + and isinstance( + getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel + ) + ) + + +def register_dsv41_o_proj_warmup(layer: nn.Module) -> None: + """Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch.""" + vllm_config = get_current_vllm_config() + if vllm_config.kernel_config.enable_jit_warmup and _can_fuse_wo_a( + layer.wo_b, + layer.n_local_groups, + layer.n_local_heads // layer.n_local_groups, + layer.nope_head_dim, + layer.rope_head_dim, + layer.o_lora_rank, + layer._einsum_recipe, + layer._tma_aligned_scales, + ): + from .fused_wo_a import _FUSED_WO_A_KERNEL + + _FUSED_WO_A_KERNEL.register_warmup( + vllm_config, + max_tokens=_FUSED_WO_A_MAX_TOKENS, + x_stride=layer.padded_heads * layer.head_dim, + ) + + +def dsv41_o_proj( + o: torch.Tensor, + positions: torch.Tensor, + cos_sin_cache: torch.Tensor, + wo_a: nn.Module, + wo_b: Callable, + *, + wo_b_layer: nn.Module, + n_groups: int, + heads_per_group: int, + nope_dim: int, + rope_dim: int, + o_lora_rank: int, + einsum_recipe: tuple[int, int, int], + tma_aligned_scales: bool, +) -> torch.Tensor: + """Fuse the BF16 attention-output path; preserve other projection paths.""" + ws = getattr(wo_a, "weight_scale", None) + if ( + 1 <= o.shape[0] <= _FUSED_WO_A_MAX_TOKENS + and ws is not None + and _can_fuse_wo_a( + wo_b_layer, + n_groups, + heads_per_group, + nope_dim, + rope_dim, + o_lora_rank, + einsum_recipe, + tma_aligned_scales, + ) + and (o.dtype, positions.dtype, cos_sin_cache.dtype, wo_a.weight.dtype, ws.dtype) + == ( + torch.bfloat16, + torch.int64, + torch.float32, + torch.float8_e4m3fn, + torch.int32, + ) + and (o.shape[1:], cos_sin_cache.shape[1:], wo_a.weight.shape, ws.shape) + == ((16, 512), (64,), (2, 1024, 4096), (2, 1024, 32)) + and o.stride()[1:] == (512, 1) + and o.stride(0) % 8 == 0 + and o.storage_offset() % 8 == 0 + and positions.stride() == (1,) + and cos_sin_cache.is_contiguous() + and wo_a.weight.is_contiguous() + and ws.stride() == (32768, 1, 1024) + ): + from .fused_wo_a import _FUSED_WO_A_KERNEL + + q, scales = _FUSED_WO_A_KERNEL( + x=o, + positions=positions, + rope=cos_sin_cache, + weight=wo_a.weight, + weight_scale=ws, + ) + return wo_b( + QuantizedActivation( + q, scales, o.dtype, torch.Size((o.shape[0], 2048)), kMxfp8Dynamic + ) + ) + return deep_gemm_fp8_o_proj( + o, + positions, + cos_sin_cache, + wo_a, + wo_b, + n_groups=n_groups, + heads_per_group=heads_per_group, + nope_dim=nope_dim, + rope_dim=rope_dim, + o_lora_rank=o_lora_rank, + einsum_recipe=einsum_recipe, + tma_aligned_scales=tma_aligned_scales, + ) From fdf2b4f04181c8f6d1637ada0ffab4cc8e962d25 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 24 Sep 2026 23:26:18 +0000 Subject: [PATCH 02/10] [Perf][DSv4.1] Take the attention layer in dsv41_o_proj; fuse up to 32 tokens - dsv41_o_proj reads its configuration from the attention layer, so the static eligibility check is shared by dispatch and warmup registration and the backend _o_proj bodies become one call. - Raise the fused range to 32 tokens, the kernel's limit: it is still 6-12% faster end to end at 26-32 tokens. - Warm every token count from 1 to the cap. Target graphs, DSpark draft graphs and eager batches land on different counts, and draft models load outside the warmup registry, so capture sizes alone missed shapes. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Juntian Liu --- .../deepseek_v41/nvidia/flashinfer_sparse.py | 34 +---- vllm/models/deepseek_v41/nvidia/flashmla.py | 17 +-- .../deepseek_v41/nvidia/ops/fused_wo_a.py | 20 +-- vllm/models/deepseek_v41/nvidia/ops/o_proj.py | 135 +++++------------- 4 files changed, 44 insertions(+), 162 deletions(-) diff --git a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py index 358dcc78f79e..0e02b486a517 100644 --- a/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py +++ b/vllm/models/deepseek_v41/nvidia/flashinfer_sparse.py @@ -243,22 +243,7 @@ def get_padded_num_q_heads(cls, num_heads: int) -> int: return _pad_to_supported_q_heads(num_heads) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return dsv41_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - wo_b_layer=self.wo_b, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) @@ -626,22 +611,7 @@ def get_padded_num_q_heads(cls, num_heads: int) -> int: return _pad_to_supported_q_heads(num_heads) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return dsv41_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - wo_b_layer=self.wo_b, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) def __init__(self, vllm_config: VllmConfig, *args, **kwargs) -> None: super().__init__(vllm_config, *args, **kwargs) diff --git a/vllm/models/deepseek_v41/nvidia/flashmla.py b/vllm/models/deepseek_v41/nvidia/flashmla.py index 0b7ecb5f12e1..ce3047ceca2b 100644 --- a/vllm/models/deepseek_v41/nvidia/flashmla.py +++ b/vllm/models/deepseek_v41/nvidia/flashmla.py @@ -61,22 +61,7 @@ def __init__(self, *args, **kwargs) -> None: register_dsv41_o_proj_warmup(self) def _o_proj(self, attn_out: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: - o = attn_out[:, : self.n_local_heads, :] - return dsv41_o_proj( - o, - positions, - self.rotary_emb.cos_sin_cache, - self.wo_a, - self._wo_b_proj, - wo_b_layer=self.wo_b, - n_groups=self.n_local_groups, - heads_per_group=self.n_local_heads // self.n_local_groups, - nope_dim=self.nope_head_dim, - rope_dim=self.rope_head_dim, - o_lora_rank=self.o_lora_rank, - einsum_recipe=self._einsum_recipe, - tma_aligned_scales=self._tma_aligned_scales, - ) + return dsv41_o_proj(self, attn_out, positions) @classmethod def get_padded_num_q_heads(cls, num_heads: int) -> int: diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py index 19e42585ff11..074ac6da5068 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -27,7 +27,7 @@ from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy from vllm.cute_utils.cvt import bf16x2_to_fp32x2, fp32x4_to_fp8x4 -from vllm.model_executor.warmup.jit_warmup import kernel_launcher +from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, kernel_launcher from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( CuTeDSLLaunchSpec, VllmCuTeDSLJitKernel, @@ -302,21 +302,11 @@ def dispatch( # type: ignore[override] ) -> CompileKey: return self.CompileKey(tokens=tokens, x_stride=x_stride) - def get_warmup_keys( - self, vllm_config: Any, *, max_tokens: int, x_stride: int - ) -> list[CompileKey]: - compilation_config = vllm_config.compilation_config - # Graphs pad batches up to a capture size; eager batches keep theirs. - padded = 0 - if compilation_config.cudagraph_mode.mixed_mode(): - padded = compilation_config.max_cudagraph_capture_size or 0 - captured = compilation_config.cudagraph_capture_sizes or [] + def get_warmup_keys(self, *, max_tokens: int, x_stride: int) -> list[CompileKey]: + # Target graphs, draft graphs and eager batches use different token + # counts, so warm every count the dispatcher accepts. return self._trace_dispatch(self.dispatch)( - tokens=[ - *(size for size in captured if size <= min(padded, max_tokens)), - *range(padded + 1, max_tokens + 1), - ], - x_stride=x_stride, + tokens=WarmupIntRange(1, max_tokens + 1), x_stride=x_stride ) def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py index 279f4658181e..431c42bd4aab 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -2,12 +2,9 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project """DSV4.1 output projection with a small-batch SM100 fusion.""" -from collections.abc import Callable - import torch from torch import nn -from vllm.config import get_current_vllm_config from vllm.model_executor.kernels.linear.mxfp8.flashinfer import ( FlashInferCutedslMxfp8LinearKernel, ) @@ -20,131 +17,71 @@ from vllm.platforms import current_platform from vllm.utils.flashinfer import has_flashinfer_cutedsl -# GB200: gains hold through 24 tokens; larger graphs have marginal benefit. -_FUSED_WO_A_MAX_TOKENS = 24 +_FUSED_WO_A_MAX_TOKENS = 32 -def _can_fuse_wo_a( - wo_b_layer: nn.Module, - n_groups: int, - heads_per_group: int, - nope_dim: int, - rope_dim: int, - o_lora_rank: int, - einsum_recipe: tuple[int, int, int], - tma_aligned_scales: bool, -) -> bool: - method = getattr(wo_b_layer, "scheme", getattr(wo_b_layer, "quant_method", None)) +def _can_fuse_wo_a(layer: nn.Module) -> bool: + """Whether the attention layer matches the fused WO-A kernel's layout.""" + method = getattr(layer.wo_b, "scheme", getattr(layer.wo_b, "quant_method", None)) + wo_b_kernel = getattr(method, "kernel", None) return ( - (n_groups, heads_per_group, nope_dim, rope_dim, o_lora_rank) - == (2, 8, 448, 64, 1024) - and einsum_recipe == (1, 1, 32) - and tma_aligned_scales - and current_platform.is_device_capability_family(100) + current_platform.is_device_capability_family(100) and has_flashinfer_cutedsl() - and get_input_quant_key(wo_b_layer) == kMxfp8Dynamic - and isinstance( - getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel - ) + and (layer.n_local_groups, layer.n_local_heads) == (2, 16) + and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64) + and layer.o_lora_rank == 1024 + and layer._einsum_recipe == (1, 1, 32) + and layer._tma_aligned_scales + and layer.wo_a.weight.dtype == torch.float8_e4m3fn + and hasattr(layer.wo_a, "weight_scale") + and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic + and isinstance(wo_b_kernel, FlashInferCutedslMxfp8LinearKernel) ) def register_dsv41_o_proj_warmup(layer: nn.Module) -> None: """Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch.""" - vllm_config = get_current_vllm_config() - if vllm_config.kernel_config.enable_jit_warmup and _can_fuse_wo_a( - layer.wo_b, - layer.n_local_groups, - layer.n_local_heads // layer.n_local_groups, - layer.nope_head_dim, - layer.rope_head_dim, - layer.o_lora_rank, - layer._einsum_recipe, - layer._tma_aligned_scales, - ): + # Draft models load outside the warmup registry; their layers share the + # target's head padding, so the target's registration covers them too. + if _can_fuse_wo_a(layer): from .fused_wo_a import _FUSED_WO_A_KERNEL _FUSED_WO_A_KERNEL.register_warmup( - vllm_config, max_tokens=_FUSED_WO_A_MAX_TOKENS, x_stride=layer.padded_heads * layer.head_dim, ) def dsv41_o_proj( - o: torch.Tensor, - positions: torch.Tensor, - cos_sin_cache: torch.Tensor, - wo_a: nn.Module, - wo_b: Callable, - *, - wo_b_layer: nn.Module, - n_groups: int, - heads_per_group: int, - nope_dim: int, - rope_dim: int, - o_lora_rank: int, - einsum_recipe: tuple[int, int, int], - tma_aligned_scales: bool, + layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor ) -> torch.Tensor: - """Fuse the BF16 attention-output path; preserve other projection paths.""" - ws = getattr(wo_a, "weight_scale", None) - if ( - 1 <= o.shape[0] <= _FUSED_WO_A_MAX_TOKENS - and ws is not None - and _can_fuse_wo_a( - wo_b_layer, - n_groups, - heads_per_group, - nope_dim, - rope_dim, - o_lora_rank, - einsum_recipe, - tma_aligned_scales, - ) - and (o.dtype, positions.dtype, cos_sin_cache.dtype, wo_a.weight.dtype, ws.dtype) - == ( - torch.bfloat16, - torch.int64, - torch.float32, - torch.float8_e4m3fn, - torch.int32, - ) - and (o.shape[1:], cos_sin_cache.shape[1:], wo_a.weight.shape, ws.shape) - == ((16, 512), (64,), (2, 1024, 4096), (2, 1024, 32)) - and o.stride()[1:] == (512, 1) - and o.stride(0) % 8 == 0 - and o.storage_offset() % 8 == 0 - and positions.stride() == (1,) - and cos_sin_cache.is_contiguous() - and wo_a.weight.is_contiguous() - and ws.stride() == (32768, 1, 1024) - ): + """``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100 batches.""" + o = attn_out[:, : layer.n_local_heads, :] + cos_sin_cache = layer.rotary_emb.cos_sin_cache + if 1 <= o.shape[0] <= _FUSED_WO_A_MAX_TOKENS and _can_fuse_wo_a(layer): from .fused_wo_a import _FUSED_WO_A_KERNEL q, scales = _FUSED_WO_A_KERNEL( x=o, positions=positions, rope=cos_sin_cache, - weight=wo_a.weight, - weight_scale=ws, + weight=layer.wo_a.weight, + weight_scale=layer.wo_a.weight_scale, ) - return wo_b( - QuantizedActivation( - q, scales, o.dtype, torch.Size((o.shape[0], 2048)), kMxfp8Dynamic - ) + return layer._wo_b_proj( + QuantizedActivation(q, scales, o.dtype, q.shape, kMxfp8Dynamic) ) return deep_gemm_fp8_o_proj( o, positions, cos_sin_cache, - wo_a, - wo_b, - n_groups=n_groups, - heads_per_group=heads_per_group, - nope_dim=nope_dim, - rope_dim=rope_dim, - o_lora_rank=o_lora_rank, - einsum_recipe=einsum_recipe, - tma_aligned_scales=tma_aligned_scales, + layer.wo_a, + layer._wo_b_proj, + n_groups=layer.n_local_groups, + heads_per_group=layer.n_local_heads // layer.n_local_groups, + nope_dim=layer.nope_head_dim, + rope_dim=layer.rope_head_dim, + o_lora_rank=layer.o_lora_rank, + einsum_recipe=layer._einsum_recipe, + tma_aligned_scales=layer._tma_aligned_scales, ) From f6f4b6863048248125ee030bc77e2bc2f755a086 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Fri, 25 Sep 2026 00:43:07 +0000 Subject: [PATCH 03/10] [Bugfix][DSv4.1] Keep fused WO-A compiling under quack's const_expr-if rewrite quack monkey-patches the CuTe DSL preprocessor process-wide on import and turns `if const_expr(S) or D:` into a constexpr if. In a vLLM server quack is always imported, so for token counts where the input stage takes the dynamic path (3, 5, 6, 7, 9, ...) `amax`, `exponent` and `inv` stay unbound after it while the output stage still yields them, and warmup fails with TYPE_UNSTABLE_JOIN. Bind them before the output stage. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Juntian Liu --- vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py index 074ac6da5068..fc674bafbf94 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -222,6 +222,10 @@ def device_kernel( owned = (tokens + 7 - split) // 8 mbarrier.arrive_expect_tx(reduced, owned * 128 * 8 * 4) cute.arch.cluster_wait() + # quack's mixed const_expr-if rewrite (installed process-wide once + # quack is imported) can leave these unbound after the dynamic path + # above; bind them so the regions below join on one type. + amax, exponent, inv = Float32(0), Uint32(0), Float32(0) if tid < 128: acc = cute.make_rmem_tensor(acc_cols, Float32) if cutlass.const_expr(tokens == 1): From 44735347fd9ab5962b0007929cf2a9fd0709e9a2 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Fri, 25 Sep 2026 01:54:15 +0000 Subject: [PATCH 04/10] [Perf][DSv4.1] Mention SM103 in the fused WO-A docstrings Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Juntian Liu --- vllm/models/deepseek_v41/nvidia/ops/o_proj.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py index 431c42bd4aab..62de6f264b23 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -"""DSV4.1 output projection with a small-batch SM100 fusion.""" +"""DSV4.1 output projection with a small-batch SM100/SM103 fusion.""" import torch from torch import nn @@ -55,7 +55,7 @@ def register_dsv41_o_proj_warmup(layer: nn.Module) -> None: def dsv41_o_proj( layer: nn.Module, attn_out: torch.Tensor, positions: torch.Tensor ) -> torch.Tensor: - """``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100 batches.""" + """``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches.""" o = attn_out[:, : layer.n_local_heads, :] cos_sin_cache = layer.rotary_emb.cos_sin_cache if 1 <= o.shape[0] <= _FUSED_WO_A_MAX_TOKENS and _can_fuse_wo_a(layer): From 2ea0e22a47bd084dba3f6fdf7f91c37ad0a7e93d Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Fri, 25 Sep 2026 07:23:48 +0000 Subject: [PATCH 05/10] [Perf][DSv4.1] Generalize fused WO-A layouts and pipeline its stages - Key the kernel on (tokens, n_groups, heads_per_group, o_lora_rank) so any TP layout with at most 8 heads per group (one portable cluster) and a 128-multiple o_lora_rank can fuse, not only 16 heads / 2 groups. - Give each 128-K stage its own mbarrier so the MMA warp starts on a stage as soon as its activations are quantized, and trigger dependent launch on the last stage. - Load weight scales with cp.async, allocate TMEM after griddepcontrol.wait so a PDL predecessor holding TMEM cannot deadlock the kernel, and let one warp wait on the DSMEM reduction. - Cover 2 and 4 groups in the kernel test. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Juntian Liu --- .../kernels/test_fused_inv_rope_fp8_quant.py | 32 +- .../deepseek_v41/nvidia/ops/fused_wo_a.py | 306 ++++++++++++------ vllm/models/deepseek_v41/nvidia/ops/o_proj.py | 24 +- 3 files changed, 239 insertions(+), 123 deletions(-) diff --git a/tests/kernels/test_fused_inv_rope_fp8_quant.py b/tests/kernels/test_fused_inv_rope_fp8_quant.py index ab8b31ef2a05..02978214f174 100644 --- a/tests/kernels/test_fused_inv_rope_fp8_quant.py +++ b/tests/kernels/test_fused_inv_rope_fp8_quant.py @@ -33,9 +33,10 @@ EPS = 1e-10 +@pytest.mark.parametrize("groups", [2, 4]) @pytest.mark.parametrize("tokens", [1, 3, 6, 8, 12, 16, 18, 32]) @pytest.mark.parametrize("padded,zero", [(False, False), (True, False), (True, True)]) -def test_dsv41_fused_wo_a_matches_deepgemm(tokens, padded, zero): +def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens, padded, zero): """Preserve per-32 WO-A math and WO-B scales, including graph replay.""" from vllm.platforms import current_platform @@ -51,19 +52,22 @@ def test_dsv41_fused_wo_a_matches_deepgemm(tokens, padded, zero): from vllm.utils.deep_gemm import fp8_einsum torch.manual_seed(42) + heads, n = groups * 8, groups * 1024 x = torch.randn( - tokens, 64 if padded else 16, 512, device="cuda", dtype=torch.bfloat16 - )[:, :16] - x.mul_(torch.exp2(torch.arange(16, device="cuda") - 8)[None, :, None]) + tokens, 64 if padded else heads, 512, device="cuda", dtype=torch.bfloat16 + )[:, :heads] + x.mul_(torch.exp2(torch.arange(heads, device="cuda") % 16 - 8)[None, :, None]) if zero: x.zero_() positions = torch.randint(0, 128, (tokens,), device="cuda") rope = make_cos_sin_cache(128, device="cuda") - w = torch.randn(2048, 4096, device="cuda", dtype=torch.bfloat16) - wb = w.float().view(2048, 128, 32) + w = torch.randn(n, 4096, device="cuda", dtype=torch.bfloat16) + wb = w.float().view(n, 128, 32) scale = torch.exp2(torch.ceil(torch.log2(wb.abs().amax(-1) / 448))) - wq = (wb / scale[..., None]).to(FP8_DTYPE).view(2048, 4096) - wq, ws = deepgemm_post_process_fp8_weight_block(wq, scale, (1, 32), False, True, 2) + wq = (wb / scale[..., None]).to(FP8_DTYPE).view(n, 4096) + wq, ws = deepgemm_post_process_fp8_weight_block( + wq, scale, (1, 32), False, True, groups + ) inputs = dict(x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws) _FUSED_WO_A_KERNEL(**inputs) graph = torch.cuda.CUDAGraph() @@ -76,21 +80,21 @@ def test_dsv41_fused_wo_a_matches_deepgemm(tokens, padded, zero): x, positions, rope, - n_groups=2, + n_groups=groups, heads_per_group=8, nope_dim=448, rope_dim=64, quant_group_size=32, tma_aligned_scales=True, ) - y = torch.empty(tokens, 2, 1024, device="cuda", dtype=torch.bfloat16) + y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16) fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32)) ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl") def dequant(data, scales): - scales = scales.reshape(16, 32, 4, 4).permute(1, 2, 0, 3) - scales = scales.reshape(128, 64)[:tokens].float() - return data.float().reshape(tokens, 64, 32) * torch.exp2( + scales = scales.reshape(n // 128, 32, 4, 4).permute(1, 2, 0, 3) + scales = scales.reshape(128, n // 32)[:tokens].float() + return data.float().reshape(tokens, n // 32, 32) * torch.exp2( scales[..., None] - 127 ) @@ -103,7 +107,7 @@ def dequant(data, scales): assert error < 0.003, error.item() rows = torch.arange(512, device="cuda") rows = rows // 16 + (rows % 16 // 4) * 32 - assert torch.count_nonzero(sf.view(16, 512)[:, rows >= tokens]) == 0 + assert torch.count_nonzero(sf.view(n // 128, 512)[:, rows >= tokens]) == 0 # ========================================================================= diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py index fc674bafbf94..f67312c41bd1 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -33,6 +33,11 @@ VllmCuTeDSLJitKernel, ) +# Each head is [nope | rope] as in fused_inv_rope_fp8_quant: only the last +# _ROPE_DIM columns carry RoPE. +_HEAD_DIM = 512 +_ROPE_DIM = 64 + @dsl_user_op def _ue8m0_rp(value, *, loc=None, ip=None): @@ -62,16 +67,30 @@ class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]): @dataclass(frozen=True) class CompileKey: tokens: int - x_stride: int + n_groups: int + heads_per_group: int + o_lora_rank: int @staticmethod def kernel(compile_key: CompileKey) -> Any: tokens = compile_key.tokens acc_cols = 1 << (tokens - 1).bit_length() tile_m = max(8, acc_cols) + tile_n = 128 + tile_k = 128 + # Each group's heads split K across one cluster, one head per CTA. + heads_per_group = compile_key.heads_per_group + tiles_per_group = compile_key.o_lora_rank // tile_n + n_tiles = compile_key.n_groups * tiles_per_group + # Each CTA keeps one whole head of K in SMEM, one TMA box per stage. + num_stages = _HEAD_DIM // tile_k + w_stage_bytes = tile_n * tile_k + nope_dim = _HEAD_DIM - _ROPE_DIM + threads_per_token = _HEAD_DIM // 4 sf_col = max(16, tile_m) tmem_cols = sf_col * 2 - threads = min(acc_cols * 128, 512) + threads = min(acc_cols * threads_per_token, 512) + tokens_per_iter = threads // threads_per_token @cute.kernel def device_kernel( @@ -87,83 +106,124 @@ def device_kernel( bid, _, _ = cute.arch.block_idx() warp = cute.arch.make_warp_uniform(tid // 32) lane = tid % 32 - split = bid % 8 - tile = bid // 8 - group = tile // 8 + split = bid % heads_per_group + tile = bid // heads_per_group + group = tile // tiles_per_group smem = utils.SmemAllocator() - sw = smem.allocate_tensor( + sW = smem.allocate_tensor( Float8E4M3FN, w.smem_layout.outer, byte_alignment=128, swizzle=w.smem_layout.inner, ) - sx = smem.allocate_tensor( + sX = smem.allocate_tensor( Float8E4M3FN, - cute.make_layout((tile_m, 128, 4), stride=(128, 1, tile_m * 128)), + cute.make_layout( + (tile_m, tile_k, num_stages), stride=(tile_k, 1, tile_m * tile_k) + ), byte_alignment=1024, ) - ssw = smem.allocate_tensor(Int32, cute.make_layout((128, 4)), 128) - ssx = smem.allocate_tensor(Int32, cute.make_layout((128, 4)), 128) + sW_SF = smem.allocate_tensor( + Int32, cute.make_layout((tile_n, num_stages)), 128 + ) + sX_SF = smem.allocate_tensor( + Int32, cute.make_layout((tile_n, num_stages)), 128 + ) partial = smem.allocate_tensor( - Float32, cute.make_layout((128, (tokens + 7) // 8, 8)), 128 + Float32, + cute.make_layout( + (tile_n, cute.ceil_div(tokens, heads_per_group), heads_per_group) + ), + 128, ) - loaded = smem.allocate_array(Int64, 4) - done = smem.allocate_array(Int64, 1) - reduced = smem.allocate_array(Int64, 1) + mbar_loaded = smem.allocate_array(Int64, num_stages) + mbar_ws_loaded = smem.allocate_array(Int64, 1) + mbar_done = smem.allocate_array(Int64, 1) + mbar_reduced = smem.allocate_array(Int64, 1) taddr = smem.allocate(Int32, 4) if tid == 0: - for stage in cutlass.range_constexpr(4): - cute.arch.mbarrier_init(loaded + stage, 1) - cute.arch.mbarrier_init(done, 1) - cute.arch.mbarrier_init(reduced, 1) + for stage in cutlass.range_constexpr(num_stages): + # Wait for weight TMA and this stage's quantization threads. + cute.arch.mbarrier_init(mbar_loaded + stage, 1 + threads // 4) + cute.arch.mbarrier_init(mbar_ws_loaded, tile_n) + cute.arch.mbarrier_init(mbar_done, 1) + cute.arch.mbarrier_init(mbar_reduced, 1) cute.arch.mbarrier_init_fence() + # Local users of the barriers (warp 0's TMA below) need the init too. cute.arch.sync_threads() + # Paired with cluster_wait() before the DSMEM exchange, so peers see + # mbar_reduced initialized before their st.async. The init fence + # above orders the init before this relaxed arrive, and splitting + # arrive / wait hides the cluster barrier behind the prologue. cute.arch.cluster_arrive_relaxed() if warp == 0: - tiles = cute.zipped_divide(w.tma_tensor, (128, 128)) - for stage in cutlass.range_constexpr(4): + tiles = cute.zipped_divide(w.tma_tensor, (tile_n, tile_k)) + for stage in cutlass.range_constexpr(num_stages): with cute.arch.elect_one(): - mbarrier.arrive_expect_tx(loaded + stage, 16384) + mbarrier.arrive_expect_tx(mbar_loaded + stage, w_stage_bytes) simple_tma_copy( w.atom, - tiles[None, (tile, split * 4 + stage)], - sw[None, None, stage], - loaded + stage, + tiles[None, (tile, split * num_stages + stage)], + sW[None, None, stage], + mbar_loaded + stage, ) - elif warp == 1: - cute.arch.alloc_tmem(tmem_cols, taddr) - cute.arch.relinquish_tmem_alloc_permit() - if tid < 128: - for stage in cutlass.range_constexpr(4): - ssw[(tid % 32) * 4 + tid // 32, stage] = ws[ - group, (tile % 8) * 128 + tid, split * 4 + stage - ] - ssx[tid, stage] = Int32(0) - sx32 = cute.recast_tensor(sx, Int32) + if tid < tile_n: + for stage in cutlass.range_constexpr(num_stages): + # TMA cannot scatter 4-byte scales into tcgen05.cp + # 32x128b.warpx4 order; a 4-byte cp.async can, without + # stalling the thread on the load. + cute.arch.cp_async_shared_global( + cute.domain_offset( + ((tid % 32) * 4 + tid // 32, stage), sW_SF + ).iterator, + cute.domain_offset( + ( + group, + (tile % tiles_per_group) * tile_n + tid, + split * num_stages + stage, + ), + ws, + ).iterator, + 4, + "ca", + ) + sX_SF[tid, stage] = Int32(0) + sX32 = cute.recast_tensor(sX, Int32) for row in cutlass.range_constexpr(tile_m // 4): - sx32[tid // 32 + row * 4, tid % 32, stage] = Int32(0) + sX32[tid // 32 + row * 4, tid % 32, stage] = Int32(0) + cute.arch.cp_async_mbarrier_arrive_noinc(mbar_ws_loaded) + # Order the zeroing before other threads' quantized stores. cute.arch.sync_threads() + # x and positions come from the PDL predecessor; everything above + # only touches weights. cute.arch.griddepcontrol_wait() + # TMEM is allocated at runtime, and a PDL predecessor on this SM may + # hold it until it exits; allocating earlier can deadlock. + if warp == 0: + cute.arch.alloc_tmem(tmem_cols, taddr) + cute.arch.relinquish_tmem_alloc_permit() x64 = cute.recast_tensor(x, Uint64) - sx32 = cute.recast_tensor(sx, Uint32) - sfbytes = cute.recast_tensor(ssx, Uint8) + sX32 = cute.recast_tensor(sX, Uint32) + sX_SF_bytes = cute.recast_tensor(sX_SF, Uint8) for iteration in cutlass.range_constexpr( - (tokens * 128 + threads - 1) // threads + cute.ceil_div(tokens, tokens_per_iter) ): - token = iteration * (threads // 128) + tid // 128 - qid = tid % 128 + token = iteration * tokens_per_iter + tid // threads_per_token + qid = (tid + 32) % threads_per_token k = qid * 4 - if cutlass.const_expr(tokens % (threads // 128) == 0) or token < tokens: - packed = x64[token, group * 8 + split, qid] + if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: + packed = x64[token, group * heads_per_group + split, qid] v0, v1 = bf16x2_to_fp32x2(Uint32(packed)) v2, v3 = bf16x2_to_fp32x2(Uint32(packed >> 32)) - if k >= 448: + if k >= nope_dim: + # Inverse RoPE on the interleaved pairs (k, k+1) and + # (k+2, k+3); each rope row is cos || sin. pos = positions[token] - c = Float32(rope[pos, (k - 448) // 2]) - s = Float32(rope[pos, (k - 448) // 2 + 32]) + c = Float32(rope[pos, (k - nope_dim) // 2]) + s = Float32(rope[pos, (k - nope_dim) // 2 + _ROPE_DIM // 2]) v0, v1 = v0 * c + v1 * s, v1 * c - v0 * s - c = Float32(rope[pos, (k - 448) // 2 + 1]) - s = Float32(rope[pos, (k - 448) // 2 + 33]) + c = Float32(rope[pos, (k - nope_dim) // 2 + 1]) + s = Float32(rope[pos, (k - nope_dim) // 2 + _ROPE_DIM // 2 + 1]) v2, v3 = v2 * c + v3 * s, v3 * c - v2 * s amax = cute.arch.fmax( cute.arch.fmax(cute.abs(v0), cute.abs(v1)), @@ -172,36 +232,41 @@ def device_kernel( amax = cute.arch.warp_reduction_max(amax, threads_in_group=8) exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10))) # Recasting drops the pointer swizzle; address it explicitly. - sx32[token, (qid % 32) ^ ((token % 8) * 4), qid // 32] = ( + sX32[token, (qid % 32) ^ ((token % 8) * 4), qid // 32] = ( fp32x4_to_fp8x4(v0 * inv, v1 * inv, v2 * inv, v3 * inv) ) if lane % 8 == 0: - sfbytes[token * 16 + k % 128 // 32 + k // 128 * 512] = Uint8( - exponent - ) - cute.arch.sync_threads() + sX_SF_bytes[ + token * 16 + k % tile_k // 32 + k // tile_k * 512 + ] = Uint8(exponent) cute.arch.fence_proxy("async.shared", space="cta") - _tcgen05.fence_after_thread_sync() - if tid == 0: - cute.arch.griddepcontrol_launch_dependents() - base = cute.make_tensor(taddr, cute.make_layout(1))[0] + mbarrier.arrive(mbar_loaded + qid // 32, order="release") if warp == 0: + base = cute.make_tensor(taddr, cute.make_layout(1))[0] sdesc = _tcgen05.make_sdesc_128B_swizzle(0) sfdesc = Uint64((8 << 32) | (1 << 46)) - idesc = _tcgen05.make_mxfp8_idesc(128, tile_m) - for stage in cutlass.range_constexpr(4): - cute.arch.mbarrier_wait(loaded + stage, 0) - adesc = sdesc | (sw[None, None, stage].iterator.toint() >> 4) - bdesc = sdesc | (sx[None, None, stage].iterator.toint() >> 4) + idesc = _tcgen05.make_mxfp8_idesc(tile_n, tile_m) + # cp.async writes through the generic proxy; tcgen05.cp reads + # through the async proxy. + cute.arch.mbarrier_wait(mbar_ws_loaded, 0) + cute.arch.fence_proxy("async.shared", space="cta") + for stage in cutlass.range_constexpr(num_stages): + cute.arch.mbarrier_wait(mbar_loaded + stage, 0) + _tcgen05.fence_after_thread_sync() + if cutlass.const_expr(stage == num_stages - 1): + with cute.arch.elect_one(): + cute.arch.griddepcontrol_launch_dependents() + adesc = sdesc | (sW[None, None, stage].iterator.toint() >> 4) + bdesc = sdesc | (sX[None, None, stage].iterator.toint() >> 4) _tcgen05.cp( base + sf_col, - sfdesc | (ssw[None, stage].iterator.toint() >> 4), + sfdesc | (sW_SF[None, stage].iterator.toint() >> 4), "32x128b", "warpx4", ) _tcgen05.cp( base + sf_col + 4, - sfdesc | (ssx[None, stage].iterator.toint() >> 4), + sfdesc | (sX_SF[None, stage].iterator.toint() >> 4), "32x128b", "warpx4", ) @@ -215,18 +280,27 @@ def device_kernel( base + sf_col + 4, stage > 0 or kk > 0, ) - _tcgen05.commit(done) - cute.arch.mbarrier_wait(done, 0) + _tcgen05.commit(mbar_done) + # Only the MMA warp waits; the CTA barrier releases the rest. + cute.arch.mbarrier_wait(mbar_done, 0) + cute.arch.sync_threads() + base = cute.make_tensor(taddr, cute.make_layout(1))[0] _tcgen05.fence_after_thread_sync() + # Tokens t with t % heads_per_group == split arrive from every peer + # (this CTA included), tile_n Float32 rows each. if tid == 0: - owned = (tokens + 7 - split) // 8 - mbarrier.arrive_expect_tx(reduced, owned * 128 * 8 * 4) + owned = cute.ceil_div(tokens - split, heads_per_group) + mbarrier.arrive_expect_tx( + mbar_reduced, owned * tile_n * heads_per_group * 4 + ) + # Pairs with cluster_arrive_relaxed() above: every peer's + # mbar_reduced is initialized before the st.async below. cute.arch.cluster_wait() # quack's mixed const_expr-if rewrite (installed process-wide once # quack is imported) can leave these unbound after the dynamic path # above; bind them so the regions below join on one type. amax, exponent, inv = Float32(0), Uint32(0), Float32(0) - if tid < 128: + if tid < tile_n: acc = cute.make_rmem_tensor(acc_cols, Float32) if cutlass.const_expr(tokens == 1): acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1) @@ -234,30 +308,44 @@ def device_kernel( acc.store(_tcgen05.ld(warp * 32, base, "32x32b", acc_cols)) _tcgen05.wait_ld() for token in cutlass.range_constexpr(tokens): - ptr = cute.domain_offset((tid, token // 8, split), partial).iterator + ptr = cute.domain_offset( + (tid, token // heads_per_group, split), partial + ).iterator cute.arch.store_async_dsmem( - ptr, recast_val(acc[token], Int32), reduced, token % 8 + ptr, + recast_val(acc[token], Int32), + mbar_reduced, + token % heads_per_group, ) if split < tokens: - cute.arch.mbarrier_wait(reduced, 0) - for i in cutlass.range_constexpr((tokens + 7) // 8): - token = split + i * 8 + # One warp waits for the peers' bytes; named barrier 1 (0 is + # sync_threads) releases the other tile_n epilogue threads. + if warp == 0: + cute.arch.mbarrier_wait(mbar_reduced, 0) + cute.arch.barrier(barrier_id=1, number_of_threads=tile_n) + for i in cutlass.range_constexpr( + cute.ceil_div(tokens, heads_per_group) + ): + token = split + i * heads_per_group if ( - cutlass.const_expr(tokens <= 8 or tokens % 8 == 0) + cutlass.const_expr( + tokens <= heads_per_group + or tokens % heads_per_group == 0 + ) or token < tokens ): value = Float32(0) - for peer in cutlass.range_constexpr(8): + for peer in cutlass.range_constexpr(heads_per_group): value += partial[tid, i, peer] value = Float32(BFloat16(value)) amax = cute.arch.warp_reduction_max(cute.abs(value)) exponent, inv = _scale(amax) - q[token, tile * 128 + tid] = Float8E4M3FN(value * inv) + q[token, tile * tile_n + tid] = Float8E4M3FN(value * inv) if lane == 0: qs[tile * 512 + token * 16 + warp] = Uint8(exponent) # Every scale tile owns its padding, with no second memset kernel. for i in cutlass.range_constexpr(4): - offset = i * 128 + tid + offset = i * tile_n + tid row = offset // 16 + (offset % 16 // 4) * 32 if split == 0 and row >= tokens: qs[tile * 512 + offset] = Uint8(0) @@ -283,18 +371,20 @@ def host_entrypoint( layout = cute.make_composed_layout( cute.make_swizzle(3, 4, 3), 0, - cute.make_layout((128, 128, 4), stride=(128, 1, 16384)), + cute.make_layout( + (tile_n, tile_k, num_stages), stride=(tile_k, 1, w_stage_bytes) + ), ) tma = cpasync.make_tiled_tma_atom( cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), w, layout, - (128, 128), + (tile_n, tile_k), ) device_kernel(x, positions, rope, tma, ws, q, qs).launch( - grid=(128, 1, 1), + grid=(n_tiles * heads_per_group, 1, 1), block=(threads, 1, 1), - cluster=(8, 1, 1), + cluster=(heads_per_group, 1, 1), stream=stream, use_pdl=True, ) @@ -302,32 +392,47 @@ def host_entrypoint( return host_entrypoint def dispatch( # type: ignore[override] - self, *, tokens: int, x_stride: int + self, *, tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int ) -> CompileKey: - return self.CompileKey(tokens=tokens, x_stride=x_stride) + return self.CompileKey( + tokens=tokens, + n_groups=n_groups, + heads_per_group=heads_per_group, + o_lora_rank=o_lora_rank, + ) - def get_warmup_keys(self, *, max_tokens: int, x_stride: int) -> list[CompileKey]: + def get_warmup_keys( + self, *, max_tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int + ) -> list[CompileKey]: # Target graphs, draft graphs and eager batches use different token # counts, so warm every count the dispatcher accepts. return self._trace_dispatch(self.dispatch)( - tokens=WarmupIntRange(1, max_tokens + 1), x_stride=x_stride + tokens=WarmupIntRange(1, max_tokens + 1), + n_groups=n_groups, + heads_per_group=heads_per_group, + o_lora_rank=o_lora_rank, ) def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: - tokens = compile_key.tokens + tokens, groups = compile_key.tokens, compile_key.n_groups + rank = compile_key.o_lora_rank + n = groups * rank + k = compile_key.heads_per_group * _HEAD_DIM return ( make_fake_tensor( BFloat16, - (tokens, 16, 512), - (compile_key.x_stride, 512, 1), + (tokens, groups * compile_key.heads_per_group, _HEAD_DIM), + (cute.sym_int(divisibility=8), _HEAD_DIM, 1), assumed_align=16, ), make_fake_tensor(Int64, (tokens,), (1,)), - make_fake_tensor(Float32, (cute.sym_int(), 64), (64, 1)), - make_fake_tensor(Float8E4M3FN, (2048, 4096), (4096, 1), assumed_align=16), - make_fake_tensor(Int32, (2, 1024, 32), (32768, 1, 1024)), - make_fake_tensor(Float8E4M3FN, (tokens, 2048), (2048, 1)), - make_fake_tensor(Uint8, (8192,), (1,)), + make_fake_tensor(Float32, (cute.sym_int(), _ROPE_DIM), (_ROPE_DIM, 1)), + make_fake_tensor(Float8E4M3FN, (n, k), (k, 1), assumed_align=16), + make_fake_tensor( + Int32, (groups, rank, k // 128), (rank * k // 128, 1, rank) + ), + make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)), + make_fake_tensor(Uint8, (128 * n // 32,), (1,)), ) @kernel_launcher @@ -342,18 +447,25 @@ def __call__( ) -> CuTeDSLLaunchSpec["FusedWoAKernel.CompileKey"]: """Return FP8 WO-A output and FlashInfer F8_128x4 scale bytes.""" tokens = x.shape[0] - q = torch.empty((tokens, 2048), device=x.device, dtype=torch.float8_e4m3fn) - scales = torch.empty(8192, device=x.device, dtype=torch.uint8) + groups, rank = weight_scale.shape[:2] + n = groups * rank + q = torch.empty((tokens, n), device=x.device, dtype=torch.float8_e4m3fn) + scales = torch.empty(128 * n // 32, device=x.device, dtype=torch.uint8) launch_args = ( x, positions, rope, - weight.view(2048, 4096), + weight.view(n, -1), weight_scale, q, scales, ) - compile_key = self.dispatch(tokens=tokens, x_stride=x.stride(0)) + compile_key = self.dispatch( + tokens=tokens, + n_groups=groups, + heads_per_group=x.shape[1] // groups, + o_lora_rank=rank, + ) return compile_key, launch_args, (q, scales) diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py index 62de6f264b23..c25365be6a4c 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -15,7 +15,6 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import kMxfp8Dynamic from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj from vllm.platforms import current_platform -from vllm.utils.flashinfer import has_flashinfer_cutedsl _FUSED_WO_A_MAX_TOKENS = 32 @@ -23,32 +22,33 @@ def _can_fuse_wo_a(layer: nn.Module) -> bool: """Whether the attention layer matches the fused WO-A kernel's layout.""" method = getattr(layer.wo_b, "scheme", getattr(layer.wo_b, "quant_method", None)) - wo_b_kernel = getattr(method, "kernel", None) return ( current_platform.is_device_capability_family(100) - and has_flashinfer_cutedsl() - and (layer.n_local_groups, layer.n_local_heads) == (2, 16) - and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64) - and layer.o_lora_rank == 1024 + # WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly. and layer._einsum_recipe == (1, 1, 32) - and layer._tma_aligned_scales - and layer.wo_a.weight.dtype == torch.float8_e4m3fn - and hasattr(layer.wo_a, "weight_scale") and get_input_quant_key(layer.wo_b) == kMxfp8Dynamic - and isinstance(wo_b_kernel, FlashInferCutedslMxfp8LinearKernel) + and isinstance( + getattr(method, "kernel", None), FlashInferCutedslMxfp8LinearKernel + ) + # A group's heads form one portable (<= 8 CTA) cluster. + and layer.n_local_heads // layer.n_local_groups <= 8 + and (layer.nope_head_dim, layer.rope_head_dim) == (448, 64) + and layer.o_lora_rank % 128 == 0 ) def register_dsv41_o_proj_warmup(layer: nn.Module) -> None: """Warm the fused WO-A token counts ``dsv41_o_proj`` can dispatch.""" # Draft models load outside the warmup registry; their layers share the - # target's head padding, so the target's registration covers them too. + # target's attention shapes, so the target's registration covers them too. if _can_fuse_wo_a(layer): from .fused_wo_a import _FUSED_WO_A_KERNEL _FUSED_WO_A_KERNEL.register_warmup( max_tokens=_FUSED_WO_A_MAX_TOKENS, - x_stride=layer.padded_heads * layer.head_dim, + n_groups=layer.n_local_groups, + heads_per_group=layer.n_local_heads // layer.n_local_groups, + o_lora_rank=layer.o_lora_rank, ) From be7dacb61570c6ea36571046800d7b2803fd9008 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Fri, 25 Sep 2026 09:03:52 +0000 Subject: [PATCH 06/10] clean code and use TMA Signed-off-by: Juntian Liu --- .../deepseek_v41/nvidia/ops/fused_wo_a.py | 178 ++++++++++-------- 1 file changed, 104 insertions(+), 74 deletions(-) diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py index f67312c41bd1..e4c103cdd6a8 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -26,7 +26,6 @@ from cutlass.cutlass_dsl import dsl_user_op from vllm.cute_utils import _tcgen05, mbarrier, recast_val, simple_tma_copy -from vllm.cute_utils.cvt import bf16x2_to_fp32x2, fp32x4_to_fp8x4 from vllm.model_executor.warmup.jit_warmup import WarmupIntRange, kernel_launcher from vllm.model_executor.warmup.jit_warmup_cutedsl_helper import ( CuTeDSLLaunchSpec, @@ -78,6 +77,8 @@ def kernel(compile_key: CompileKey) -> Any: tile_m = max(8, acc_cols) tile_n = 128 tile_k = 128 + # tcgen05 MXFP8 MMA consumes K = 32, one MX block, per instruction. + mma_k = 32 # Each group's heads split K across one cluster, one head per CTA. heads_per_group = compile_key.heads_per_group tiles_per_group = compile_key.o_lora_rank // tile_n @@ -98,7 +99,7 @@ def device_kernel( positions: cute.Tensor, rope: cute.Tensor, w: cpasync.TmaInfo, - ws: cute.Tensor, + ws: cpasync.TmaInfo, q: cute.Tensor, qs: cute.Tensor, ): @@ -122,12 +123,28 @@ def device_kernel( (tile_m, tile_k, num_stages), stride=(tile_k, 1, tile_m * tile_k) ), byte_alignment=1024, + swizzle=cute.make_swizzle(3, 4, 3), ) + # tcgen05.cp 32x128b.warpx4 order: row r at byte (r % 32) * 16 + + # (r // 32) * 4, one 4-byte cell of MX-block scales per row. sW_SF = smem.allocate_tensor( + Int32, + cute.make_layout(((32, 4), num_stages), stride=((4, 1), tile_n)), + 128, + ) + sW_SF_raw = smem.allocate_tensor( Int32, cute.make_layout((tile_n, num_stages)), 128 ) sX_SF = smem.allocate_tensor( - Int32, cute.make_layout((tile_n, num_stages)), 128 + Uint8, + cute.make_layout( + ((32, 4), tile_k // mma_k, num_stages), + stride=((16, 4), 1, tile_n * 4), + ), + 128, + ) + copy_fp8x4 = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float8E4M3FN, num_bits_per_copy=32 ) partial = smem.allocate_tensor( Float32, @@ -145,18 +162,27 @@ def device_kernel( for stage in cutlass.range_constexpr(num_stages): # Wait for weight TMA and this stage's quantization threads. cute.arch.mbarrier_init(mbar_loaded + stage, 1 + threads // 4) - cute.arch.mbarrier_init(mbar_ws_loaded, tile_n) + cute.arch.mbarrier_init(mbar_ws_loaded, 1) cute.arch.mbarrier_init(mbar_done, 1) cute.arch.mbarrier_init(mbar_reduced, 1) cute.arch.mbarrier_init_fence() # Local users of the barriers (warp 0's TMA below) need the init too. cute.arch.sync_threads() - # Paired with cluster_wait() before the DSMEM exchange, so peers see - # mbar_reduced initialized before their st.async. The init fence - # above orders the init before this relaxed arrive, and splitting - # arrive / wait hides the cluster barrier behind the prologue. + # Paired with cluster_wait() below: peers must see mbar_reduced + # initialized before their st.async; the split hides the barrier. cute.arch.cluster_arrive_relaxed() if warp == 0: + # Raw DeepGEMM MN-major scales; the MMA warp transposes them to + # the UTCCP order, which TMA cannot scatter at 4-byte granularity. + ws_tiles = cute.zipped_divide(ws.tma_tensor, (tile_n, num_stages)) + with cute.arch.elect_one(): + mbarrier.arrive_expect_tx(mbar_ws_loaded, tile_n * num_stages * 4) + simple_tma_copy( + ws.atom, + ws_tiles[None, (tile % tiles_per_group, split, group)], + sW_SF_raw, + mbar_ws_loaded, + ) tiles = cute.zipped_divide(w.tma_tensor, (tile_n, tile_k)) for stage in cutlass.range_constexpr(num_stages): with cute.arch.elect_one(): @@ -167,89 +193,83 @@ def device_kernel( sW[None, None, stage], mbar_loaded + stage, ) - if tid < tile_n: - for stage in cutlass.range_constexpr(num_stages): - # TMA cannot scatter 4-byte scales into tcgen05.cp - # 32x128b.warpx4 order; a 4-byte cp.async can, without - # stalling the thread on the load. - cute.arch.cp_async_shared_global( - cute.domain_offset( - ((tid % 32) * 4 + tid // 32, stage), sW_SF - ).iterator, - cute.domain_offset( - ( - group, - (tile % tiles_per_group) * tile_n + tid, - split * num_stages + stage, - ), - ws, - ).iterator, - 4, - "ca", - ) - sX_SF[tid, stage] = Int32(0) - sX32 = cute.recast_tensor(sX, Int32) - for row in cutlass.range_constexpr(tile_m // 4): - sX32[tid // 32 + row * 4, tid % 32, stage] = Int32(0) - cute.arch.cp_async_mbarrier_arrive_noinc(mbar_ws_loaded) - # Order the zeroing before other threads' quantized stores. - cute.arch.sync_threads() # x and positions come from the PDL predecessor; everything above - # only touches weights. + # only touches weights. Padded sX rows only feed unread MMA columns. cute.arch.griddepcontrol_wait() # TMEM is allocated at runtime, and a PDL predecessor on this SM may # hold it until it exits; allocating earlier can deadlock. if warp == 0: cute.arch.alloc_tmem(tmem_cols, taddr) cute.arch.relinquish_tmem_alloc_permit() - x64 = cute.recast_tensor(x, Uint64) - sX32 = cute.recast_tensor(sX, Uint32) - sX_SF_bytes = cute.recast_tensor(sX_SF, Uint8) - for iteration in cutlass.range_constexpr( - cute.ceil_div(tokens, tokens_per_iter) - ): + copy_x = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), BFloat16, num_bits_per_copy=64 + ) + n_iters = cute.ceil_div(tokens, tokens_per_iter) + qid = (tid + 32) % threads_per_token + k = qid * 4 + # Issue every token's x load before any math so their latencies overlap. + x_bf16 = cute.make_rmem_tensor((4, n_iters), BFloat16) + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: + x_src = cute.local_tile( + x[token, group * heads_per_group + split, None], (4,), (qid,) + ) + cute.copy(copy_x, x_src, x_bf16[None, iteration]) + for iteration in cutlass.range_constexpr(n_iters): token = iteration * tokens_per_iter + tid // threads_per_token - qid = (tid + 32) % threads_per_token - k = qid * 4 if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: - packed = x64[token, group * heads_per_group + split, qid] - v0, v1 = bf16x2_to_fp32x2(Uint32(packed)) - v2, v3 = bf16x2_to_fp32x2(Uint32(packed >> 32)) + x_f32 = cute.make_rmem_tensor((4,), Float32) + x_f32.store(x_bf16[None, iteration].load().to(Float32)) if k >= nope_dim: # Inverse RoPE on the interleaved pairs (k, k+1) and # (k+2, k+3); each rope row is cos || sin. pos = positions[token] - c = Float32(rope[pos, (k - nope_dim) // 2]) - s = Float32(rope[pos, (k - nope_dim) // 2 + _ROPE_DIM // 2]) - v0, v1 = v0 * c + v1 * s, v1 * c - v0 * s - c = Float32(rope[pos, (k - nope_dim) // 2 + 1]) - s = Float32(rope[pos, (k - nope_dim) // 2 + _ROPE_DIM // 2 + 1]) - v2, v3 = v2 * c + v3 * s, v3 * c - v2 * s + for pair in cutlass.range_constexpr(2): + freq = (k - nope_dim) // 2 + pair + cos = Float32(rope[pos, freq]) + sin = Float32(rope[pos, freq + _ROPE_DIM // 2]) + even, odd = x_f32[2 * pair], x_f32[2 * pair + 1] + x_f32[2 * pair] = even * cos + odd * sin + x_f32[2 * pair + 1] = odd * cos - even * sin amax = cute.arch.fmax( - cute.arch.fmax(cute.abs(v0), cute.abs(v1)), - cute.arch.fmax(cute.abs(v2), cute.abs(v3)), + cute.arch.fmax(cute.abs(x_f32[0]), cute.abs(x_f32[1])), + cute.arch.fmax(cute.abs(x_f32[2]), cute.abs(x_f32[3])), ) amax = cute.arch.warp_reduction_max(amax, threads_in_group=8) exponent, inv = _scale(cute.arch.fmax(amax, Float32(1e-10))) - # Recasting drops the pointer swizzle; address it explicitly. - sX32[token, (qid % 32) ^ ((token % 8) * 4), qid // 32] = ( - fp32x4_to_fp8x4(v0 * inv, v1 * inv, v2 * inv, v3 * inv) + x_fp8 = cute.make_rmem_tensor((4,), Float8E4M3FN) + x_fp8.store((x_f32.load() * inv).to(Float8E4M3FN)) + cute.copy( + copy_fp8x4, + x_fp8, + cute.local_tile( + sX[token, None, k // tile_k], (4,), (qid % 32,) + ), ) if lane % 8 == 0: - sX_SF_bytes[ - token * 16 + k % tile_k // 32 + k // tile_k * 512 - ] = Uint8(exponent) + sX_SF[token, k % tile_k // mma_k, k // tile_k] = Uint8(exponent) cute.arch.fence_proxy("async.shared", space="cta") mbarrier.arrive(mbar_loaded + qid // 32, order="release") if warp == 0: base = cute.make_tensor(taddr, cute.make_layout(1))[0] - sdesc = _tcgen05.make_sdesc_128B_swizzle(0) - sfdesc = Uint64((8 << 32) | (1 << 46)) + # sW / sX are 128B-swizzled K-major; one stage's K fills the + # swizzle atom, so the leading byte offset is unused. + sdesc = _tcgen05.make_sdesc_128B_swizzle(LBO=0) + # Unswizzled 32 x 16 B scale tiles: SBO = one 8-row core matrix + # (16 B units at bit 32); bit 46 is the SM100 descriptor version. + sf_sbo = 8 * 16 + sfdesc = Uint64((sf_sbo >> 4 << 32) | (1 << 46)) + # M = tile_n weight rows (A), N = tile_m token rows (B). idesc = _tcgen05.make_mxfp8_idesc(tile_n, tile_m) - # cp.async writes through the generic proxy; tcgen05.cp reads - # through the async proxy. + # sW_SF's layout is the UTCCP order, so a plain copy transposes; + # fence the generic-proxy writes before tcgen05.cp reads them. cute.arch.mbarrier_wait(mbar_ws_loaded, 0) + for stage in cutlass.range_constexpr(num_stages): + for i in cutlass.range_constexpr(4): + sW_SF[i * 32 + lane, stage] = sW_SF_raw[i * 32 + lane, stage] cute.arch.fence_proxy("async.shared", space="cta") + cute.arch.sync_warp() for stage in cutlass.range_constexpr(num_stages): cute.arch.mbarrier_wait(mbar_loaded + stage, 0) _tcgen05.fence_after_thread_sync() @@ -266,15 +286,17 @@ def device_kernel( ) _tcgen05.cp( base + sf_col + 4, - sfdesc | (sX_SF[None, stage].iterator.toint() >> 4), + sfdesc | (sX_SF[None, None, stage].iterator.toint() >> 4), "32x128b", "warpx4", ) - for kk in cutlass.range_constexpr(4): + # Advance mma_k FP8 bytes (16 B units) and pick MX block kk of + # each 4-byte scale (B sf_id at bit 4, A sf_id at bit 29). + for kk in cutlass.range_constexpr(tile_k // mma_k): _tcgen05.mma_mxfp8( base, - adesc + kk * 2, - bdesc + kk * 2, + adesc + kk * mma_k // 16, + bdesc + kk * mma_k // 16, idesc + ((kk << 4) | (kk << 29)), base + sf_col, base + sf_col + 4, @@ -296,9 +318,8 @@ def device_kernel( # Pairs with cluster_arrive_relaxed() above: every peer's # mbar_reduced is initialized before the st.async below. cute.arch.cluster_wait() - # quack's mixed const_expr-if rewrite (installed process-wide once - # quack is imported) can leave these unbound after the dynamic path - # above; bind them so the regions below join on one type. + # quack's process-wide const_expr-if rewrite can leave these unbound + # after the dynamic path above; bind them so the regions join. amax, exponent, inv = Float32(0), Uint32(0), Float32(0) if tid < tile_n: acc = cute.make_rmem_tensor(acc_cols, Float32) @@ -381,7 +402,13 @@ def host_entrypoint( layout, (tile_n, tile_k), ) - device_kernel(x, positions, rope, tma, ws, q, qs).launch( + ws_tma = cpasync.make_tiled_tma_atom( + cpasync.CopyBulkTensorTileG2SOp(cta_group=tcgen05.CtaGroup.ONE), + cute.make_tensor(ws.iterator, cute.select(ws.layout, mode=[1, 2, 0])), + cute.make_layout((tile_n, num_stages)), + (tile_n, num_stages), + ) + device_kernel(x, positions, rope, tma, ws_tma, q, qs).launch( grid=(n_tiles * heads_per_group, 1, 1), block=(threads, 1, 1), cluster=(heads_per_group, 1, 1), @@ -429,7 +456,10 @@ def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: make_fake_tensor(Float32, (cute.sym_int(), _ROPE_DIM), (_ROPE_DIM, 1)), make_fake_tensor(Float8E4M3FN, (n, k), (k, 1), assumed_align=16), make_fake_tensor( - Int32, (groups, rank, k // 128), (rank * k // 128, 1, rank) + Int32, + (groups, rank, k // 128), + (rank * k // 128, 1, rank), + assumed_align=16, ), make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)), make_fake_tensor(Uint8, (128 * n // 32,), (1,)), From e63dfaec759aac23d9b2635187b443bcb0ecf8dd Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Fri, 25 Sep 2026 19:07:22 +0000 Subject: [PATCH 07/10] clean code Signed-off-by: Juntian Liu --- .../kernels/test_fused_inv_rope_fp8_quant.py | 99 +++++++------------ vllm/models/deepseek_v41/nvidia/ops/o_proj.py | 2 +- 2 files changed, 36 insertions(+), 65 deletions(-) diff --git a/tests/kernels/test_fused_inv_rope_fp8_quant.py b/tests/kernels/test_fused_inv_rope_fp8_quant.py index 02978214f174..3b5b24fe49f9 100644 --- a/tests/kernels/test_fused_inv_rope_fp8_quant.py +++ b/tests/kernels/test_fused_inv_rope_fp8_quant.py @@ -21,7 +21,12 @@ import pytest import torch +from vllm.model_executor.layers.quantization.utils.fp8_utils import ( + deepgemm_post_process_fp8_weight_block, +) from vllm.models.deepseek_v4.common.ops import fused_inv_rope_fp8_quant +from vllm.platforms import current_platform +from vllm.utils.deep_gemm import fp8_einsum # -- Default dimensions matching DeepSeek V3/V4 -------------------------- HEAD_DIM = 512 @@ -33,81 +38,47 @@ EPS = 1e-10 +@pytest.mark.skipif( + not current_platform.is_device_capability_family(100), reason="Requires SM100/SM103" +) @pytest.mark.parametrize("groups", [2, 4]) -@pytest.mark.parametrize("tokens", [1, 3, 6, 8, 12, 16, 18, 32]) -@pytest.mark.parametrize("padded,zero", [(False, False), (True, False), (True, True)]) -def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens, padded, zero): - """Preserve per-32 WO-A math and WO-B scales, including graph replay.""" - from vllm.platforms import current_platform - - if not current_platform.is_device_capability_family(100): - pytest.skip("Requires SM100/SM103") - pytest.importorskip("cutlass") +@pytest.mark.parametrize("tokens", [1, 6, 18, 32]) +def test_dsv41_fused_wo_a_matches_deepgemm(groups, tokens): + """Fused WO-A matches inverse RoPE + fp8_einsum + MXFP8 quantize.""" from flashinfer import mxfp8_quantize - from vllm.model_executor.layers.quantization.utils.fp8_utils import ( - deepgemm_post_process_fp8_weight_block, - ) from vllm.models.deepseek_v41.nvidia.ops.fused_wo_a import _FUSED_WO_A_KERNEL - from vllm.utils.deep_gemm import fp8_einsum - torch.manual_seed(42) - heads, n = groups * 8, groups * 1024 - x = torch.randn( - tokens, 64 if padded else heads, 512, device="cuda", dtype=torch.bfloat16 - )[:, :heads] - x.mul_(torch.exp2(torch.arange(heads, device="cuda") % 16 - 8)[None, :, None]) - if zero: - x.zero_() + torch.manual_seed(0) + n = groups * 1024 + x = torch.randn(tokens, 64, 512, device="cuda", dtype=torch.bfloat16)[:, : n // 128] positions = torch.randint(0, 128, (tokens,), device="cuda") rope = make_cos_sin_cache(128, device="cuda") - w = torch.randn(n, 4096, device="cuda", dtype=torch.bfloat16) - wb = w.float().view(n, 128, 32) - scale = torch.exp2(torch.ceil(torch.log2(wb.abs().amax(-1) / 448))) - wq = (wb / scale[..., None]).to(FP8_DTYPE).view(n, 4096) + wq = torch.randn(n, 4096, device="cuda").to(FP8_DTYPE) + scale = torch.exp2(torch.randint(-8, 0, (n, 128), device="cuda").float()) wq, ws = deepgemm_post_process_fp8_weight_block( wq, scale, (1, 32), False, True, groups ) - inputs = dict(x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws) - _FUSED_WO_A_KERNEL(**inputs) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - q, sf = _FUSED_WO_A_KERNEL(**inputs) - for _ in range(2): - positions.add_(1).remainder_(128) - graph.replay() - a, sa = fused_inv_rope_fp8_quant( - x, - positions, - rope, - n_groups=groups, - heads_per_group=8, - nope_dim=448, - rope_dim=64, - quant_group_size=32, - tma_aligned_scales=True, - ) - y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16) - fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32)) - ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl") - - def dequant(data, scales): - scales = scales.reshape(n // 128, 32, 4, 4).permute(1, 2, 0, 3) - scales = scales.reshape(128, n // 32)[:tokens].float() - return data.float().reshape(tokens, n // 32, 32) * torch.exp2( - scales[..., None] - 127 - ) + q, sf = _FUSED_WO_A_KERNEL( + x=x, positions=positions, rope=rope, weight=wq, weight_scale=ws + ) - actual, expected = dequant(q, sf), dequant(ref, ref_sf) - assert torch.isfinite(actual).all() - if zero: - assert torch.count_nonzero(q.float()) == 0 - else: - error = (actual - expected).norm() / expected.norm() - assert error < 0.003, error.item() - rows = torch.arange(512, device="cuda") - rows = rows // 16 + (rows % 16 // 4) * 32 - assert torch.count_nonzero(sf.view(n // 128, 512)[:, rows >= tokens]) == 0 + a, sa = fused_inv_rope_fp8_quant( + x, positions, rope, groups, 8, quant_group_size=32, tma_aligned_scales=True + ) + y = torch.empty(tokens, groups, 1024, device="cuda", dtype=torch.bfloat16) + fp8_einsum("bhr,hdr->bhd", (a, sa), (wq, ws), y, recipe=(1, 1, 32)) + ref, ref_sf = mxfp8_quantize(y.flatten(1), backend="cute-dsl") + + def rows(scales): # F8_128x4 bytes -> [128 rows, n / 32 blocks] + return scales.view(n // 128, 32, 4, 4).permute(2, 1, 0, 3).reshape(128, -1) + + actual, expected = ( + d.float().view(tokens, -1, 32) * torch.exp2(rows(s)[:tokens, :, None] - 127.0) + for d, s in ((q, sf), (ref, ref_sf)) + ) + assert (actual - expected).norm() / expected.norm() < 3e-3 + assert not rows(sf)[tokens:].any(), "padded scale rows must be zero" # ========================================================================= diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py index c25365be6a4c..f30349316a80 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -21,7 +21,7 @@ def _can_fuse_wo_a(layer: nn.Module) -> bool: """Whether the attention layer matches the fused WO-A kernel's layout.""" - method = getattr(layer.wo_b, "scheme", getattr(layer.wo_b, "quant_method", None)) + method = getattr(layer.wo_b, "scheme", layer.wo_b.quant_method) return ( current_platform.is_device_capability_family(100) # WO-A is FP8 with per-32 scales, and WO-B takes MXFP8 input directly. From 4f59bdde4bea20ce3e031db496afb5606bc6247e Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Sat, 26 Sep 2026 00:37:29 +0000 Subject: [PATCH 08/10] further improve perf without numeric diff Signed-off-by: Juntian Liu --- .../deepseek_v41/nvidia/ops/fused_wo_a.py | 89 ++++++++++++++++--- 1 file changed, 76 insertions(+), 13 deletions(-) diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py index e4c103cdd6a8..404d49097872 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -53,6 +53,26 @@ def _ue8m0_rp(value, *, loc=None, ip=None): return Uint32(result) +@dsl_user_op +def _rope_fma(value, cos, other, sin, *, subtract=False, loc=None, ip=None): + """Preserve multiply/FMA rounding when RoPE loads are hoisted.""" + result = llvm.inline_asm( + Float32.mlir_type, + [v.ir_value(loc=loc, ip=ip) for v in (value, cos, other, sin)], + "{ .reg .f32 product; mul.rn.f32 product, $3, $4; " + + ( + "neg.f32 product, product; fma.rn.f32 $0, $1, $2, product; }" + if subtract + else "fma.rn.f32 $0, $1, $2, product; }" + ), + "=f,f,f,f,f", + has_side_effects=False, + loc=loc, + ip=ip, + ) + return Float32(result) + + @cute.jit def _scale(amax: Float32): exponent = _ue8m0_rp(amax * Float32(1.0 / 448.0)) @@ -216,6 +236,36 @@ def device_kernel( x[token, group * heads_per_group + split, None], (4,), (qid,) ) cute.copy(copy_x, x_src, x_bf16[None, iteration]) + # Overlap the RoPE loads across tokens before consuming any of them. + cos_reg = cute.make_rmem_tensor((2, n_iters), Float32) + sin_reg = cute.make_rmem_tensor((2, n_iters), Float32) + copy_rope = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32 + ) + if cutlass.const_expr(n_iters > 1): # noqa: SIM102 + if k >= nope_dim: + for iteration in cutlass.range_constexpr(n_iters): + token = iteration * tokens_per_iter + tid // threads_per_token + if ( + cutlass.const_expr(tokens % tokens_per_iter == 0) + or token < tokens + ): + pos = positions[token] + freq = (k - nope_dim) // 2 + cute.copy( + copy_rope, + cute.local_tile(rope[pos, None], (2,), (freq // 2,)), + cos_reg[None, iteration], + ) + cute.copy( + copy_rope, + cute.local_tile( + rope[pos, None], + (2,), + ((freq + _ROPE_DIM // 2) // 2,), + ), + sin_reg[None, iteration], + ) for iteration in cutlass.range_constexpr(n_iters): token = iteration * tokens_per_iter + tid // threads_per_token if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: @@ -224,14 +274,20 @@ def device_kernel( if k >= nope_dim: # Inverse RoPE on the interleaved pairs (k, k+1) and # (k+2, k+3); each rope row is cos || sin. - pos = positions[token] for pair in cutlass.range_constexpr(2): - freq = (k - nope_dim) // 2 + pair - cos = Float32(rope[pos, freq]) - sin = Float32(rope[pos, freq + _ROPE_DIM // 2]) + if cutlass.const_expr(n_iters > 1): + cos = cos_reg[pair, iteration] + sin = sin_reg[pair, iteration] + else: + pos = positions[token] + freq = (k - nope_dim) // 2 + pair + cos = Float32(rope[pos, freq]) + sin = Float32(rope[pos, freq + _ROPE_DIM // 2]) even, odd = x_f32[2 * pair], x_f32[2 * pair + 1] - x_f32[2 * pair] = even * cos + odd * sin - x_f32[2 * pair + 1] = odd * cos - even * sin + x_f32[2 * pair] = _rope_fma(even, cos, odd, sin) + x_f32[2 * pair + 1] = _rope_fma( + odd, cos, even, sin, subtract=True + ) amax = cute.arch.fmax( cute.arch.fmax(cute.abs(x_f32[0]), cute.abs(x_f32[1])), cute.arch.fmax(cute.abs(x_f32[2]), cute.abs(x_f32[3])), @@ -303,11 +359,10 @@ def device_kernel( stage > 0 or kk > 0, ) _tcgen05.commit(mbar_done) - # Only the MMA warp waits; the CTA barrier releases the rest. + # Only the MMA warp waits; barrier 2 releases the TMEM readers. cute.arch.mbarrier_wait(mbar_done, 0) - cute.arch.sync_threads() - base = cute.make_tensor(taddr, cute.make_layout(1))[0] - _tcgen05.fence_after_thread_sync() + if tid < tile_n: + cute.arch.barrier(barrier_id=2, number_of_threads=tile_n) # Tokens t with t % heads_per_group == split arrive from every peer # (this CTA included), tile_n Float32 rows each. if tid == 0: @@ -322,6 +377,8 @@ def device_kernel( # after the dynamic path above; bind them so the regions join. amax, exponent, inv = Float32(0), Uint32(0), Float32(0) if tid < tile_n: + base = cute.make_tensor(taddr, cute.make_layout(1))[0] + _tcgen05.fence_after_thread_sync() acc = cute.make_rmem_tensor(acc_cols, Float32) if cutlass.const_expr(tokens == 1): acc[0] = _tcgen05.ld(warp * 32, base, "32x32b", 1) @@ -359,7 +416,7 @@ def device_kernel( for peer in cutlass.range_constexpr(heads_per_group): value += partial[tid, i, peer] value = Float32(BFloat16(value)) - amax = cute.arch.warp_reduction_max(cute.abs(value)) + amax = cute.arch.warp_redux_sync(value, "max", abs=True) exponent, inv = _scale(amax) q[token, tile * tile_n + tid] = Float8E4M3FN(value * inv) if lane == 0: @@ -372,10 +429,16 @@ def device_kernel( qs[tile * 512 + offset] = Uint8(0) # Each receiver drained all writes into its own inbox. Only local # TMEM readers must finish before deallocation; no peer reads SMEM. - cute.arch.sync_threads() + if tid < tile_n: + cute.arch.barrier(barrier_id=2, number_of_threads=tile_n) if warp == 0: cute.arch.dealloc_tmem( - cute.make_ptr(Float32, base, cute.AddressSpace.tmem), tmem_cols + cute.make_ptr( + Float32, + cute.make_tensor(taddr, cute.make_layout(1))[0], + cute.AddressSpace.tmem, + ), + tmem_cols, ) @cute.jit From 3995dbc3da3ac036bbc5dc4bec29c99d7695ad58 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Sat, 26 Sep 2026 02:08:55 +0000 Subject: [PATCH 09/10] support larger batch size and clean code Signed-off-by: Juntian Liu --- .../deepseek_v41/nvidia/ops/fused_wo_a.py | 79 +++++++++++-------- vllm/models/deepseek_v41/nvidia/ops/o_proj.py | 10 ++- 2 files changed, 55 insertions(+), 34 deletions(-) diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py index 404d49097872..5552fec762c6 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -92,7 +92,15 @@ class CompileKey: @staticmethod def kernel(compile_key: CompileKey) -> Any: - tokens = compile_key.tokens + # Keep the 32-token tile's register footprint for larger batches. + total_tokens = compile_key.tokens + # Balance three tiles with a multiple of four tokens in each. + tokens = ( + cute.ceil_div(total_tokens, 12) * 4 + if 64 < total_tokens <= 96 + else min(total_tokens, 32) + ) + token_tiles = cute.ceil_div(total_tokens, tokens) acc_cols = 1 << (tokens - 1).bit_length() tile_m = max(8, acc_cols) tile_n = 128 @@ -124,7 +132,10 @@ def device_kernel( qs: cute.Tensor, ): tid, _, _ = cute.arch.thread_idx() - bid, _, _ = cute.arch.block_idx() + bid, token_tile, _ = cute.arch.block_idx() + token_start = Int32(0) + if cutlass.const_expr(token_tiles > 1): + token_start = token_tile * tokens warp = cute.arch.make_warp_uniform(tid // 32) lane = tid % 32 split = bid % heads_per_group @@ -227,22 +238,29 @@ def device_kernel( n_iters = cute.ceil_div(tokens, tokens_per_iter) qid = (tid + 32) % threads_per_token k = qid * 4 + # Tail rows repeat the last input; their output stores are masked. # Issue every token's x load before any math so their latencies overlap. x_bf16 = cute.make_rmem_tensor((4, n_iters), BFloat16) for iteration in cutlass.range_constexpr(n_iters): token = iteration * tokens_per_iter + tid // threads_per_token if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: x_src = cute.local_tile( - x[token, group * heads_per_group + split, None], (4,), (qid,) + x[ + cute.min(token_start + token, total_tokens - 1), + group * heads_per_group + split, + None, + ], + (4,), + (qid,), ) cute.copy(copy_x, x_src, x_bf16[None, iteration]) # Overlap the RoPE loads across tokens before consuming any of them. cos_reg = cute.make_rmem_tensor((2, n_iters), Float32) sin_reg = cute.make_rmem_tensor((2, n_iters), Float32) - copy_rope = cute.make_copy_atom( - cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32 - ) - if cutlass.const_expr(n_iters > 1): # noqa: SIM102 + if cutlass.const_expr(n_iters > 1): + copy_rope = cute.make_copy_atom( + cute.nvgpu.CopyUniversalOp(), Float32, num_bits_per_copy=32 + ) if k >= nope_dim: for iteration in cutlass.range_constexpr(n_iters): token = iteration * tokens_per_iter + tid // threads_per_token @@ -250,22 +268,15 @@ def device_kernel( cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens ): - pos = positions[token] + src_token = cute.min(token_start + token, total_tokens - 1) + pos = positions[src_token] freq = (k - nope_dim) // 2 - cute.copy( - copy_rope, - cute.local_tile(rope[pos, None], (2,), (freq // 2,)), - cos_reg[None, iteration], - ) - cute.copy( - copy_rope, - cute.local_tile( - rope[pos, None], - (2,), - ((freq + _ROPE_DIM // 2) // 2,), - ), - sin_reg[None, iteration], - ) + rope_row = rope[pos, None] + cos_src = cute.local_tile(rope_row, (2,), (freq // 2,)) + cute.copy(copy_rope, cos_src, cos_reg[None, iteration]) + sin_tile = (freq + _ROPE_DIM // 2) // 2 + sin_src = cute.local_tile(rope_row, (2,), (sin_tile,)) + cute.copy(copy_rope, sin_src, sin_reg[None, iteration]) for iteration in cutlass.range_constexpr(n_iters): token = iteration * tokens_per_iter + tid // threads_per_token if cutlass.const_expr(tokens % tokens_per_iter == 0) or token < tokens: @@ -279,7 +290,9 @@ def device_kernel( cos = cos_reg[pair, iteration] sin = sin_reg[pair, iteration] else: - pos = positions[token] + pos = positions[ + cute.min(token_start + token, total_tokens - 1) + ] freq = (k - nope_dim) // 2 + pair cos = Float32(rope[pos, freq]) sin = Float32(rope[pos, freq + _ROPE_DIM // 2]) @@ -405,27 +418,31 @@ def device_kernel( cute.ceil_div(tokens, heads_per_group) ): token = split + i * heads_per_group - if ( - cutlass.const_expr( + if cutlass.const_expr( + total_tokens % tokens == 0 + and ( tokens <= heads_per_group or tokens % heads_per_group == 0 ) - or token < tokens - ): + ) or (token < tokens and token_start + token < total_tokens): value = Float32(0) for peer in cutlass.range_constexpr(heads_per_group): value += partial[tid, i, peer] value = Float32(BFloat16(value)) amax = cute.arch.warp_redux_sync(value, "max", abs=True) exponent, inv = _scale(amax) - q[token, tile * tile_n + tid] = Float8E4M3FN(value * inv) + q[token_start + token, tile * tile_n + tid] = Float8E4M3FN( + value * inv + ) if lane == 0: - qs[tile * 512 + token * 16 + warp] = Uint8(exponent) + row = token_start + token + offset = row % 32 * 16 + row // 32 * 4 + warp + qs[tile * 512 + offset] = Uint8(exponent) # Every scale tile owns its padding, with no second memset kernel. for i in cutlass.range_constexpr(4): offset = i * tile_n + tid row = offset // 16 + (offset % 16 // 4) * 32 - if split == 0 and row >= tokens: + if token_tile == 0 and split == 0 and row >= total_tokens: qs[tile * 512 + offset] = Uint8(0) # Each receiver drained all writes into its own inbox. Only local # TMEM readers must finish before deallocation; no peer reads SMEM. @@ -472,7 +489,7 @@ def host_entrypoint( (tile_n, num_stages), ) device_kernel(x, positions, rope, tma, ws_tma, q, qs).launch( - grid=(n_tiles * heads_per_group, 1, 1), + grid=(n_tiles * heads_per_group, token_tiles, 1), block=(threads, 1, 1), cluster=(heads_per_group, 1, 1), stream=stream, diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py index f30349316a80..7aa5e23bce85 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -16,7 +16,11 @@ from vllm.models.deepseek_v4.nvidia.ops.o_proj import deep_gemm_fp8_o_proj from vllm.platforms import current_platform -_FUSED_WO_A_MAX_TOKENS = 32 + +def _fused_wo_a_max_tokens(layer: nn.Module) -> int: + # Additional token tiles only pay off for smaller grids. + ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128) + return 96 if ctas_per_tile <= 128 else 32 def _can_fuse_wo_a(layer: nn.Module) -> bool: @@ -45,7 +49,7 @@ def register_dsv41_o_proj_warmup(layer: nn.Module) -> None: from .fused_wo_a import _FUSED_WO_A_KERNEL _FUSED_WO_A_KERNEL.register_warmup( - max_tokens=_FUSED_WO_A_MAX_TOKENS, + max_tokens=_fused_wo_a_max_tokens(layer), n_groups=layer.n_local_groups, heads_per_group=layer.n_local_heads // layer.n_local_groups, o_lora_rank=layer.o_lora_rank, @@ -58,7 +62,7 @@ def dsv41_o_proj( """``deep_gemm_fp8_o_proj`` with WO-A fused for small SM100/SM103 batches.""" o = attn_out[:, : layer.n_local_heads, :] cos_sin_cache = layer.rotary_emb.cos_sin_cache - if 1 <= o.shape[0] <= _FUSED_WO_A_MAX_TOKENS and _can_fuse_wo_a(layer): + if 1 <= o.shape[0] <= _fused_wo_a_max_tokens(layer) and _can_fuse_wo_a(layer): from .fused_wo_a import _FUSED_WO_A_KERNEL q, scales = _FUSED_WO_A_KERNEL( From 813bcd4042f5883832f7c23592e9114d295504b0 Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Sat, 26 Sep 2026 02:24:03 +0000 Subject: [PATCH 10/10] reduce compile key combs Signed-off-by: Juntian Liu --- .../deepseek_v41/nvidia/ops/fused_wo_a.py | 49 +++++++++++++------ vllm/models/deepseek_v41/nvidia/ops/o_proj.py | 10 ++-- 2 files changed, 40 insertions(+), 19 deletions(-) diff --git a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py index 5552fec762c6..b195d4b84937 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py +++ b/vllm/models/deepseek_v41/nvidia/ops/fused_wo_a.py @@ -85,22 +85,19 @@ def _scale(amax: Float32): class FusedWoAKernel(VllmCuTeDSLJitKernel["FusedWoAKernel.CompileKey"]): @dataclass(frozen=True) class CompileKey: - tokens: int + # T=1..96 uses at most 46 keys per (n_groups, heads_per_group, o_lora_rank). + tokens_per_tile: int + token_tiles: int + full_tiles: bool n_groups: int heads_per_group: int o_lora_rank: int @staticmethod def kernel(compile_key: CompileKey) -> Any: - # Keep the 32-token tile's register footprint for larger batches. - total_tokens = compile_key.tokens - # Balance three tiles with a multiple of four tokens in each. - tokens = ( - cute.ceil_div(total_tokens, 12) * 4 - if 64 < total_tokens <= 96 - else min(total_tokens, 32) - ) - token_tiles = cute.ceil_div(total_tokens, tokens) + tokens = compile_key.tokens_per_tile + token_tiles = compile_key.token_tiles + full_tiles = compile_key.full_tiles acc_cols = 1 << (tokens - 1).bit_length() tile_m = max(8, acc_cols) tile_n = 128 @@ -130,7 +127,11 @@ def device_kernel( ws: cpasync.TmaInfo, q: cute.Tensor, qs: cute.Tensor, + num_tokens: Int32, ): + total_tokens = num_tokens + if cutlass.const_expr(full_tiles): + total_tokens = Int32(tokens * token_tiles) tid, _, _ = cute.arch.thread_idx() bid, token_tile, _ = cute.arch.block_idx() token_start = Int32(0) @@ -419,7 +420,7 @@ def device_kernel( ): token = split + i * heads_per_group if cutlass.const_expr( - total_tokens % tokens == 0 + full_tiles and ( tokens <= heads_per_group or tokens % heads_per_group == 0 @@ -467,6 +468,7 @@ def host_entrypoint( ws: cute.Tensor, q: cute.Tensor, qs: cute.Tensor, + num_tokens: Int32, stream: CUstream, ): layout = cute.make_composed_layout( @@ -488,7 +490,7 @@ def host_entrypoint( cute.make_layout((tile_n, num_stages)), (tile_n, num_stages), ) - device_kernel(x, positions, rope, tma, ws_tma, q, qs).launch( + device_kernel(x, positions, rope, tma, ws_tma, q, qs, num_tokens).launch( grid=(n_tiles * heads_per_group, token_tiles, 1), block=(threads, 1, 1), cluster=(heads_per_group, 1, 1), @@ -501,8 +503,17 @@ def host_entrypoint( def dispatch( # type: ignore[override] self, *, tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int ) -> CompileKey: + token_tiles = (tokens + 31) // 32 + # Host integer arithmetic: balance tiles in multiples of four tokens. + tile_tokens = ( + tokens + if token_tiles == 1 + else (tokens + 4 * token_tiles - 1) // (4 * token_tiles) * 4 + ) return self.CompileKey( - tokens=tokens, + tokens_per_tile=tile_tokens, + token_tiles=token_tiles, + full_tiles=tokens % tile_tokens == 0, n_groups=n_groups, heads_per_group=heads_per_group, o_lora_rank=o_lora_rank, @@ -511,8 +522,7 @@ def dispatch( # type: ignore[override] def get_warmup_keys( self, *, max_tokens: int, n_groups: int, heads_per_group: int, o_lora_rank: int ) -> list[CompileKey]: - # Target graphs, draft graphs and eager batches use different token - # counts, so warm every count the dispatcher accepts. + # Trace all accepted token counts and deduplicate shared tile shapes. return self._trace_dispatch(self.dispatch)( tokens=WarmupIntRange(1, max_tokens + 1), n_groups=n_groups, @@ -521,7 +531,12 @@ def get_warmup_keys( ) def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: - tokens, groups = compile_key.tokens, compile_key.n_groups + tokens = ( + compile_key.tokens_per_tile * compile_key.token_tiles + if compile_key.full_tiles + else cute.sym_int() + ) + groups = compile_key.n_groups rank = compile_key.o_lora_rank n = groups * rank k = compile_key.heads_per_group * _HEAD_DIM @@ -543,6 +558,7 @@ def warmup_inputs(self, compile_key: CompileKey) -> tuple[Any, ...]: ), make_fake_tensor(Float8E4M3FN, (tokens, n), (n, 1)), make_fake_tensor(Uint8, (128 * n // 32,), (1,)), + Int32(0), ) @kernel_launcher @@ -569,6 +585,7 @@ def __call__( weight_scale, q, scales, + tokens, ) compile_key = self.dispatch( tokens=tokens, diff --git a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py index 7aa5e23bce85..4a315ddc9ba6 100644 --- a/vllm/models/deepseek_v41/nvidia/ops/o_proj.py +++ b/vllm/models/deepseek_v41/nvidia/ops/o_proj.py @@ -18,9 +18,13 @@ def _fused_wo_a_max_tokens(layer: nn.Module) -> int: - # Additional token tiles only pay off for smaller grids. - ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128) - return 96 if ctas_per_tile <= 128 else 32 + max_tokens = getattr(layer, "_fused_wo_a_token_limit", None) + if max_tokens is None: + # Additional token tiles only pay off for smaller grids. + ctas_per_tile = layer.n_local_heads * (layer.o_lora_rank // 128) + max_tokens = 96 if ctas_per_tile <= 128 else 32 + layer._fused_wo_a_token_limit = max_tokens + return max_tokens def _can_fuse_wo_a(layer: nn.Module) -> bool: