Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
73a1c79
Added a guard to dsv_32 fused_norm_rope as ROCm Triton passes no laun…
jhu960213 Aug 10, 2026
12ed6ed
The batched gemm is already imported in aiter ops so changing to use …
jhu960213 Aug 10, 2026
d104154
Added power of 2 guard for fp4 bmm, dropped sparse MLA block size ove…
jhu960213 Aug 11, 2026
d91ae1b
Seperated vendor specific logic and kept the shared common fused kern…
jhu960213 Aug 12, 2026
70d1891
Parametrized model internal configs and used corrected values for bot…
jhu960213 Aug 12, 2026
825a834
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 12, 2026
e1810f0
Update get_supported_block_sizes() in rocm.py to prevent duplicated i…
jhu960213 Aug 12, 2026
df1e2a0
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 12, 2026
ef4c74f
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 12, 2026
9e46c41
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 12, 2026
06b5547
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 13, 2026
6bdc26e
Apply non duplicated supported block sizes to DeepseekV32IndexerBacke…
jhu960213 Aug 13, 2026
3ef331a
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 13, 2026
d4debcb
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 20, 2026
c2cb9ed
Fixed fused_norm_rope crashing on shared layers after the main merge
jhu960213 Aug 21, 2026
4a9f852
Re-resolved the deepseek_v32 fused kernel tests clobbered by the main…
jhu960213 Aug 21, 2026
5074aa2
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 21, 2026
03540d5
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 26, 2026
1280c56
Restored the ROCm branch of the deepseek_v32 entry point
jhu960213 Aug 26, 2026
c3de540
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 26, 2026
1fdc85a
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 26, 2026
3f94981
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 26, 2026
6ac6e08
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 26, 2026
bad5aa0
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 27, 2026
d10f12d
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Aug 27, 2026
6088a13
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 1, 2026
7835341
Restored the DCP no-local-slots test dropped by a main merge
jhu960213 Sep 1, 2026
a01639d
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 1, 2026
c58e8dd
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 8, 2026
b124eff
Declared the AITER sparse block sizes on the backend classes
jhu960213 Sep 8, 2026
16bf640
Dropped the fp4 bmm power-of-two guard and the W_K contiguous copy
jhu960213 Sep 8, 2026
bd3ccea
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 8, 2026
bf4544e
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 9, 2026
fcd0680
Dropped the redundant int block sizes from the DSA backends
jhu960213 Sep 9, 2026
0e6d4b1
Routed GlmMoeDsaForCausalLM to the AMD DSA module on ROCm
jhu960213 Sep 9, 2026
38eecbb
Folded USE_PDL into the PDL launch kwargs
jhu960213 Sep 9, 2026
8200224
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 9, 2026
90b15a9
Merge remote-tracking branch 'origin/main' into jhu96/enable-glm52-in…
jhu960213 Sep 9, 2026
870eeb3
Merge remote-tracking branch 'fork/jhu96/enable-glm52-in-models' into…
jhu960213 Sep 9, 2026
99226d1
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 10, 2026
e53ffcb
Added platform aware fp8 quantization for glm52 on the deepseek_v32 p…
jhu960213 Sep 14, 2026
0906144
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 14, 2026
2d5c177
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 14, 2026
c8b6979
Fixed merge conflict, added back my parametrization
jhu960213 Sep 14, 2026
998bf01
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 15, 2026
d93c177
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 15, 2026
52ab91a
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 16, 2026
39ba4e5
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 16, 2026
e8967b3
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 20, 2026
21135e8
Merge branch 'main' into jhu96/enable-glm52-in-models
dllehr-amd Sep 21, 2026
8a1175d
[Bugfix][Test] Fix pre-commit: parametrize nvfp4 ds_mla test by model…
dllehr-amd Sep 21, 2026
933d45c
Merge branch 'main' into jhu96/enable-glm52-in-models
jhu960213 Sep 21, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
216 changes: 126 additions & 90 deletions tests/kernels/test_fused_deepseek_v32_norm_rope.py

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion vllm/_aiter_ops.py
Original file line number Diff line number Diff line change
Expand Up @@ -2853,7 +2853,7 @@ def batched_gemm_a16wfp4(
X: torch.Tensor,
W: torch.Tensor,
w_scale: torch.Tensor,
Y: torch.Tensor,
Y: torch.Tensor | None = None,
transpose_bm: bool | None = False,
prequant: bool | None = False,
y_scale: torch.Tensor | None = None,
Expand Down
26 changes: 20 additions & 6 deletions vllm/model_executor/layers/attention/mla_attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@
from vllm.model_executor.utils import replace_parameter
from vllm.platforms import current_platform
from vllm.utils.flashinfer import has_flashinfer
from vllm.utils.math_utils import cdiv, round_down, round_up
from vllm.utils.math_utils import cdiv, next_power_of_2, round_down, round_up
from vllm.utils.torch_utils import (
LayerNameType,
_encode_layer_name,
Expand Down Expand Up @@ -602,12 +602,27 @@ def __init__(

self.is_aiter_triton_fp8_bmm_enabled = rocm_aiter_ops.is_fp8bmm_enabled()

# aiter fp4 bmm only supports powers of two for K, else it returns garbage
# (GLM-5.2 192, DSv3.2 128)
fp4_bmm_k_supported = (
next_power_of_2(self.qk_nope_head_dim) == self.qk_nope_head_dim
and next_power_of_2(self.kv_lora_rank) == self.kv_lora_rank
)
# If kv_b_proj_weight is unquantized, quantize it to mxfp4 if supported
self.is_aiter_triton_fp4_bmm_enabled = (
rocm_aiter_ops.is_fp4bmm_enabled()
and hasattr(self.kv_b_proj, "weight")
and self.kv_b_proj.weight.dtype == torch.bfloat16
and fp4_bmm_k_supported
)
if rocm_aiter_ops.is_fp4bmm_enabled() and not fp4_bmm_k_supported:
logger.warning_once(
"aiter fp4 bmm is disabled because qk_nope_head_dim=%d and "
"kv_lora_rank=%d are not both powers of two; using the fp8 bmm "
"instead. Set VLLM_ROCM_USE_AITER_FP8BMM=0 to fall back to torch.bmm.",
self.qk_nope_head_dim,
self.kv_lora_rank,
)

# Attributes for forward_impl method
self._vllm_config = get_current_vllm_config()
Expand Down Expand Up @@ -877,9 +892,7 @@ def forward_impl(
mqa_q_pe = mqa_pe_padded

if self.is_aiter_triton_fp4_bmm_enabled:
from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4

mqa_ql_nope = batched_gemm_a16wfp4(
mqa_ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4(
mqa_q_nope,
self.W_K,
self.W_K_scale,
Expand Down Expand Up @@ -1064,8 +1077,9 @@ def process_weights_after_loading(self, act_dtype: torch.dtype):

self.W_K, self.W_K_scale = quark_quantize_weight_to_mxfp4(W_UK)
# Convert from (L, N, P) to (N, L, P)
self.W_K = self.W_K.transpose(0, 1)
self.W_K_scale = self.W_K_scale.transpose(0, 1)
# contiguous: aiter's fp4 bmm returns NaN on strided weights
Comment thread
jhu960213 marked this conversation as resolved.
Resolved
self.W_K = self.W_K.transpose(0, 1).contiguous()
self.W_K_scale = self.W_K_scale.transpose(0, 1).contiguous()

self.W_V, self.W_V_scale = quark_quantize_weight_to_mxfp4(
W_UV.permute(1, 2, 0)
Expand Down
5 changes: 5 additions & 0 deletions vllm/model_executor/models/deepseek_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -626,6 +626,11 @@ def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec:

def forward(self): ...

@property
def uses_shuffled_layout(self) -> bool:
"""Whether this cache's reader expects the shuffled value layout."""
return False

def get_attn_backend(self) -> AttentionBackend:
return DeepseekV32IndexerBackend

Expand Down
29 changes: 17 additions & 12 deletions vllm/models/deepseek_v32/amd/rocm.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import torch

from vllm._aiter_ops import rocm_aiter_ops
from vllm.compilation.breakable_cudagraph import eager_break_during_capture
from vllm.forward_context import get_forward_context
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
Expand All @@ -16,22 +17,31 @@
)


# The aiter sparse kernels also handle 16 and 32, so extend rather than replace the
# base declaration -- narrowing it to [16, 32] made select_common_block_size silently
# downgrade a requested 64 to 32 via its largest-divisor fallback.
class DeepseekV32MLASparseBackend(ROCMAiterMLASparseBackend):
@staticmethod
def get_supported_kernel_block_sizes() -> list:
return [16, 32]
return ROCMAiterMLASparseBackend.get_supported_kernel_block_sizes() + [16, 32]
Comment thread
jhu960213 marked this conversation as resolved.
Resolved


class DeepseekV32ROCmIndexerBackend(DeepseekV32IndexerBackend):
@staticmethod
def get_supported_kernel_block_sizes() -> list:
return [16, 32]
return DeepseekV32IndexerBackend.get_supported_kernel_block_sizes() + [16, 32]


class DeepseekV32ROCmIndexerCache(DeepseekV32IndexerCache):
def get_attn_backend(self):
return DeepseekV32ROCmIndexerBackend

@property
def uses_shuffled_layout(self) -> bool:
# aiter's gather/insert pair shuffles the cache above block size 1:
# [n_blocks, blk/16, head_dim/16, 16, 16] instead of [n_blocks, blk, head_dim].
return self.kv_cache.ndim == 3 and self.kv_cache.shape[1] != 1


class DeepseekV32ROCmIndexer(DeepseekV32Indexer):
indexer_cache_cls = DeepseekV32ROCmIndexerCache
Expand Down Expand Up @@ -101,14 +111,10 @@ def _compute_ql_nope(self, q_c: torch.Tensor) -> tuple[torch.Tensor, torch.Tenso
q_nope = q_nope.transpose(0, 1) # (N, tokens, P)

if self.is_aiter_triton_fp4_bmm_enabled:
from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4

ql_nope = batched_gemm_a16wfp4(
ql_nope = rocm_aiter_ops.batched_gemm_a16wfp4(
q_nope, self.W_K, self.W_K_scale, transpose_bm=True, prequant=True
)
elif self.is_aiter_triton_fp8_bmm_enabled:
from vllm._aiter_ops import rocm_aiter_ops

ql_nope = rocm_aiter_ops.triton_fp8_bmm(
q_nope, self.W_K, self.W_K_scale, group_size=128, transpose_bm=True
)
Expand Down Expand Up @@ -153,14 +159,10 @@ def _compute_uv_out(
)

if self.is_aiter_triton_fp4_bmm_enabled:
from aiter.ops.triton.batched_gemm_a16wfp4 import batched_gemm_a16wfp4

batched_gemm_a16wfp4(
rocm_aiter_ops.batched_gemm_a16wfp4(
x, self.W_V, self.W_V_scale, out_view, transpose_bm=True, prequant=True
)
elif self.is_aiter_triton_fp8_bmm_enabled:
from vllm._aiter_ops import rocm_aiter_ops

rocm_aiter_ops.triton_fp8_bmm(
x,
self.W_V,
Expand Down Expand Up @@ -203,6 +205,7 @@ def _fused_attention(
indexer_k_norm_eps = self.indexer.k_norm.eps
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
indexer_k_cache = self.indexer.k_cache.kv_cache
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
indexer_softmax_scale = self.indexer.softmax_scale
indexer_n_head_scale = self.indexer.n_head**-0.5
else:
Expand All @@ -212,6 +215,7 @@ def _fused_attention(
indexer_k_norm_eps = 1e-6
indexer_k_rope_cos_sin_cache = None
indexer_k_cache = None
indexer_cache_shuffled = False
indexer_softmax_scale = 0.0
indexer_n_head_scale = 0.0

Expand Down Expand Up @@ -242,6 +246,7 @@ def _fused_attention(
self.topk_indices_buffer,
slot_mapping=mla_slot,
indexer_k_cache=indexer_k_cache,
indexer_cache_shuffled=indexer_cache_shuffled,
mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=self.kv_cache_dtype,
mla_k_scale=mla_k_scale,
Expand Down
3 changes: 3 additions & 0 deletions vllm/models/deepseek_v32/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,7 @@ def forward( # type: ignore[override]
indexer_k_norm_eps = self.indexer.k_norm.eps
indexer_k_rope_cos_sin_cache = self.indexer_rope_emb.cos_sin_cache
indexer_k_cache = self.indexer.k_cache.kv_cache
indexer_cache_shuffled = self.indexer.k_cache.uses_shuffled_layout
indexer_softmax_scale = self.indexer.softmax_scale
indexer_n_head_scale = self.indexer.n_head**-0.5
else:
Expand All @@ -382,6 +383,7 @@ def forward( # type: ignore[override]
indexer_k_norm_eps = 1e-6
indexer_k_rope_cos_sin_cache = None
indexer_k_cache = None
indexer_cache_shuffled = False
indexer_softmax_scale = 0.0
indexer_n_head_scale = 0.0

Expand Down Expand Up @@ -412,6 +414,7 @@ def forward( # type: ignore[override]
self.topk_indices_buffer,
slot_mapping=mla_slot,
indexer_k_cache=indexer_k_cache,
indexer_cache_shuffled=indexer_cache_shuffled,
mla_kv_cache=mla_kv_cache,
mla_kv_cache_dtype=self.kv_cache_dtype,
mla_k_scale=mla_k_scale,
Expand Down
63 changes: 55 additions & 8 deletions vllm/models/deepseek_v32/common/kernels.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,12 @@
# has_indexer=False path so the indexer args don't allocate every call.
_DUMMY_CACHE: dict[tuple, torch.Tensor] = {}

# Tile shape of the indexer-K cache's shuffled layout, used when the cache reports
# uses_shuffled_layout. The block tile is a token count; the head tile is a byte
# count, which the kernel converts to cache elements.
_INDEXER_CACHE_BLOCK_TILE = 16
_INDEXER_CACHE_HEAD_TILE_BYTES = 16


def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor:
key = (shape, dtype, device)
Expand Down Expand Up @@ -70,18 +76,30 @@ def _fp8_quant_and_cache_write(
cache_stride,
offsets,
HEAD_DIM: tl.constexpr,
SHUFFLE: tl.constexpr,
BLOCK_TILE: tl.constexpr,
HEAD_TILE: tl.constexpr,
):
k_fp8, scale = _fp8_ue8m0_quantize(vals)

block_idx = slot_idx // cache_block_size
block_offset = slot_idx % cache_block_size
block_start = block_idx * cache_block_size * cache_stride

tl.store(
kv_cache_ptr + block_start + block_offset * HEAD_DIM + offsets,
k_fp8,
mask=mask,
)
# Shuffled layout: [blk/BLOCK_TILE, HEAD_DIM/HEAD_TILE, BLOCK_TILE, HEAD_TILE],
# so one contiguous run holds BLOCK_TILE tokens x HEAD_TILE bytes and the
# reader's coalesced loads land on the bytes we wrote.
if SHUFFLE:
value_off = (
block_offset // BLOCK_TILE * BLOCK_TILE * HEAD_DIM
+ block_offset % BLOCK_TILE * HEAD_TILE
+ offsets // HEAD_TILE * BLOCK_TILE * HEAD_TILE
+ offsets % HEAD_TILE
)
else:
value_off = block_offset * HEAD_DIM + offsets

tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask)
scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4
tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale)

Expand Down Expand Up @@ -129,6 +147,9 @@ def _fused_norm_rope_kernel(
indexer_cache_scale_ptr,
indexer_cache_block_size,
indexer_cache_stride,
INDEXER_CACHE_SHUFFLE: tl.constexpr,
INDEXER_CACHE_BLOCK_TILE: tl.constexpr,
INDEXER_CACHE_HEAD_TILE: tl.constexpr,
# MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr)
mla_cache_ptr,
mla_cache_block_stride,
Expand Down Expand Up @@ -279,7 +300,7 @@ def _fused_norm_rope_kernel(
# Shared layer: no indexer K to process.
return
# Fused: Index K LayerNorm + RoPE + FP8 quant + cache write.
# Eliminates the separate indexer_k_quant_and_cache kernel launch.
# Eliminates the separate indexer-K quant-and-cache kernel launch.
Comment thread
jhu960213 marked this conversation as resolved.
Resolved

index_k_block = tl.arange(0, INDEX_K_BLOCK_SIZE)
index_k_mask = index_k_block < INDEX_K_DIM
Expand Down Expand Up @@ -367,6 +388,9 @@ def _fused_norm_rope_kernel(
indexer_cache_stride,
index_k_block,
INDEX_K_DIM,
INDEXER_CACHE_SHUFFLE,
INDEXER_CACHE_BLOCK_TILE,
INDEXER_CACHE_HEAD_TILE,
)


Expand All @@ -389,6 +413,7 @@ def fused_norm_rope(
# Cache params for fused writes (single slot_mapping for both caches)
slot_mapping: torch.Tensor | None = None,
indexer_k_cache: torch.Tensor | None = None,
indexer_cache_shuffled: bool = False,
mla_kv_cache: torch.Tensor | None = None,
mla_kv_cache_dtype: str = "auto",
mla_k_scale: torch.Tensor | None = None,
Expand Down Expand Up @@ -427,6 +452,14 @@ def fused_norm_rope(
idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32)
idx_cache_block_size = indexer_k_cache.shape[1]
idx_cache_stride = indexer_k_cache.shape[2]
# The caller's cache reports whether its reader expects the shuffled
# layout; see DeepseekV32IndexerCache.uses_shuffled_layout.
idx_cache_shuffle = indexer_cache_shuffled
if idx_cache_shuffle:
assert idx_cache_block_size % _INDEXER_CACHE_BLOCK_TILE == 0, (
f"indexer K cache block size {idx_cache_block_size} must be a "
f"multiple of {_INDEXER_CACHE_BLOCK_TILE} for the shuffled layout"
)
if indexer_k_cache.dtype == torch.uint8:
indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not necessarily related to your change, but should this be current_platform.fp8_dtype()?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This line actually predates my changes. I was thinking the same, tbh, I think it's better with current_platform.fp8_dtype(), I'll change it.

else:
Expand All @@ -436,12 +469,19 @@ def fused_norm_rope(
indexer_k_cache = torch.empty(0, dtype=torch.float8_e4m3fn, device=device)
idx_cache_block_size = 1
idx_cache_stride = 1
idx_cache_shuffle = False
if mla_kv_cache is None:
# Pure profiling run (no caches at all): skip all per-token writes.
slot_mapping = torch.full(
(num_tokens,), -1, dtype=torch.int64, device=device
)

# The head tile is a byte count and the kernel indexes indexer_k_cache in
# elements, so convert using the dtype the kernel actually receives.
idx_cache_head_tile = (
_INDEXER_CACHE_HEAD_TILE_BYTES // indexer_k_cache.element_size()
)

# --- MLA KV cache setup ---
mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla"
mla_cache_fp8 = mla_kv_cache_dtype not in ("auto", "fp8_ds_mla")
Expand Down Expand Up @@ -479,6 +519,9 @@ def fused_norm_rope(
if q_c_out is None:
q_c_out = torch.empty_like(q_c)
use_pdl = current_platform.is_arch_support_pdl()
# `launch_pdl` is a CUDA-only Triton runtime kwarg; ROCm's Triton rejects it
Comment thread
jhu960213 marked this conversation as resolved.
Resolved
# even when False, so only pass it when PDL is actually supported.
pdl_kwargs = {"launch_pdl": True} if use_pdl else {}
_fused_norm_rope_kernel[(4, num_tokens)](
positions,
# Q RMS norm
Expand Down Expand Up @@ -519,6 +562,9 @@ def fused_norm_rope(
idx_cache_scale_view,
idx_cache_block_size,
idx_cache_stride,
idx_cache_shuffle,
_INDEXER_CACHE_BLOCK_TILE,
idx_cache_head_tile,
# MLA KV cache (uses same slot_mapping)
mla_kv_cache,
mla_block_stride,
Expand All @@ -538,7 +584,7 @@ def fused_norm_rope(
HAS_INDEXER=has_indexer,
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
**pdl_kwargs,
)
return q_c_out

Expand Down Expand Up @@ -869,6 +915,7 @@ def fused_q(
return index_q_fp8, index_weights_out, mqa_q

use_pdl = current_platform.is_arch_support_pdl()
pdl_kwargs = {"launch_pdl": True} if use_pdl else {}
_fused_q_kernel[(3, num_tokens, grid_heads)](
positions,
q_pe,
Expand Down Expand Up @@ -911,7 +958,7 @@ def fused_q(
INDEX_ROPE_INTERLEAVE=index_rope_interleave,
QUANTIZE_MQA=quantize_mqa,
USE_PDL=use_pdl,
launch_pdl=use_pdl,
**pdl_kwargs,
# num_warps=1 is optimal here: each program is a single 128-element
# rope+quant, so the kernel is program-count/occupancy bound, not
# per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere).
Expand Down
Loading