Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 24 additions & 11 deletions python/sglang/kernels/ops/attention/table_qk_norm_rope.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ def _table_qk_norm_rope_kernel(
D: tl.constexpr,
EPS: tl.constexpr,
MATCH_CUDA: tl.constexpr,
BF16_TABLE: tl.constexpr,
):
t = tl.program_id(0).to(tl.int64)
h = tl.program_id(1)
Expand All @@ -26,8 +27,8 @@ def _table_qk_norm_rope_kernel(
HALF: tl.constexpr = D // 2
half_ar = tl.arange(0, HALF)
d_ar = tl.arange(0, D)
cos = tl.load(cos_sin_ptr + pos * D + half_ar).to(tl.float32)
sin = tl.load(cos_sin_ptr + pos * D + HALF + half_ar).to(tl.float32)
cos = tl.load(cos_sin_ptr + pos * D + half_ar)
sin = tl.load(cos_sin_ptr + pos * D + HALF + half_ar)

is_q = h < NHQ
col0 = tl.where(is_q, h * D, q_size + (h - NHQ) * D).to(tl.int64)
Expand All @@ -51,14 +52,23 @@ def _table_qk_norm_rope_kernel(
w2 = tl.load(w_ptr + HALF + half_ar).to(tl.float32)
x1 = tl.load(row + half_ar).to(tl.float32) * inv * w1
x2 = tl.load(row + HALF + half_ar).to(tl.float32) * inv * w2
x1 = x1.to(tl.bfloat16).to(tl.float32)
x2 = x2.to(tl.bfloat16).to(tl.float32)
if MATCH_CUDA:
o1 = tl.fma(x1, cos, -mul_rn_f32(x2, sin))
o2 = tl.fma(x2, cos, mul_rn_f32(x1, sin))
else:
x1 = x1.to(tl.bfloat16)
x2 = x2.to(tl.bfloat16)
if BF16_TABLE:
# Same BF16 expression as triton_mrope_fused, so its codegen matches.
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
else:
x1 = x1.to(tl.float32)
x2 = x2.to(tl.float32)
cos = cos.to(tl.float32)
sin = sin.to(tl.float32)
if MATCH_CUDA:
o1 = tl.fma(x1, cos, -mul_rn_f32(x2, sin))
o2 = tl.fma(x2, cos, mul_rn_f32(x1, sin))
else:
o1 = x1 * cos - x2 * sin
o2 = x2 * cos + x1 * sin
tl.store(row + half_ar, o1.to(tl.bfloat16))
tl.store(row + HALF + half_ar, o2.to(tl.bfloat16))

Expand All @@ -85,8 +95,9 @@ def table_qk_norm_rope_(
``match_cuda`` preserves the reduction and rounding of the separate CUDA
QK-norm and RoPE kernels for head dimensions 64/128/256. It requires BF16
QKV with contiguous columns, contiguous BF16 weights, a contiguous FP32
rotary table, and contiguous int32/int64 positions on the same CUDA device.
Row-strided QKV is supported.
or BF16 rotary table, and contiguous int32/int64 positions on the same CUDA
device. Row-strided QKV is supported. A BF16 table rotates in BF16 to match
``triton_mrope_fused`` on decode positions, where all three axes are equal.
"""
if match_cuda:
assert head_dim in (64, 128, 256)
Expand All @@ -96,7 +107,8 @@ def table_qk_norm_rope_(
assert q_weight.shape == k_weight.shape == (head_dim,)
assert q_weight.is_contiguous() and k_weight.is_contiguous()
assert cos_sin_cache.ndim == 2 and cos_sin_cache.shape[1] == head_dim
assert cos_sin_cache.dtype == torch.float32 and cos_sin_cache.is_contiguous()
assert cos_sin_cache.dtype in (torch.float32, torch.bfloat16)
assert cos_sin_cache.is_contiguous()
assert positions.shape == (qkv.shape[0],) and positions.is_contiguous()
assert positions.dtype in (torch.int32, torch.int64)
assert qkv.is_cuda and all(
Expand All @@ -119,5 +131,6 @@ def table_qk_norm_rope_(
D=head_dim,
EPS=eps,
MATCH_CUDA=match_cuda,
BF16_TABLE=cos_sin_cache.dtype == torch.bfloat16,
num_warps=1 if match_cuda else 4,
)
17 changes: 16 additions & 1 deletion python/sglang/srt/models/qwen3.py
Original file line number Diff line number Diff line change
Expand Up @@ -262,9 +262,18 @@ def __init__(
and not envs.SGLANG_ENABLE_DETERMINISTIC_INFERENCE.get()
and get_exec().deterministic.rl_on_policy_target is None
and get_exec().graph.cuda_graph_config.prefill.tc_compiler != "inductor"
and type(self.rotary_emb) is RotaryEmbedding
and (
type(self.rotary_emb) is RotaryEmbedding
or (
type(self.rotary_emb) is MRotaryEmbedding
and self.rotary_emb.mrope_section is not None
)
)
and self.head_dim in (64, 128, 256)
)
self.table_rope_positions_ndim = (
2 if type(self.rotary_emb) is MRotaryEmbedding else 1
)

self.use_fused_qk_norm_mrope = (
_has_fused_qk_norm_mrope
Expand Down Expand Up @@ -309,7 +318,13 @@ def forward_prepare_native(self, positions, hidden_states, forward_batch):
and qkv.dtype == torch.bfloat16
and self.q_norm.weight.dtype == torch.bfloat16
and self.k_norm.weight.dtype == torch.bfloat16
and positions.ndim == self.table_rope_positions_ndim
):
if positions.ndim == 2:

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The fused path relies on _compute_mrope_positions_decode producing repeated positions. Its fallback path (has_precomputed_mrope) produces real spatial positions where the axes differ. If another model built on Qwen3Attention ever supplies precomputed decode positions, and the flag is on, it will produce silently wrong output, with no crash.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed. A silently wrong fallback isn't acceptable. Fixed in b39e69b and eee8d67 by removing the assumption instead of documenting it.

The kernel now takes the full [3, T] MRoPE positions plus MRotaryEmbedding.axis_map, and gathers each rotary lane's position from its own axis. triton_mrope_fused does the same with its section masks, and axis_map encodes those masks exactly, including the interleaved and GLM-interleaved layouts. The result is correct for any decode positions, including precomputed spatial ones.

Tests now use distinct positions on each axis:

  • Kernel: bit-exact against JIT QK-norm + triton_mrope_fused for the section, interleaved, and GLM-interleaved layouts.
  • Model: the dispatch and CUDA-graph replay tests feed 2D positions with different rows.

# Decode MRoPE repeats one position on all three axes; GLM-Image's
# precomputed spatial decode positions don't reach Qwen3Attention.
positions = positions[0]
self.rotary_emb._match_cos_sin_cache_dtype(qkv)
table_qk_norm_rope_(
qkv,
positions,
Expand Down
23 changes: 22 additions & 1 deletion test/registered/kernels/ops/attention/test_table_qk_norm_rope.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import torch

from sglang.kernels.ops.attention.rope import apply_rope_inplace
from sglang.kernels.ops.attention.rotary_triton import triton_mrope_fused
from sglang.kernels.ops.attention.table_qk_norm_rope import table_qk_norm_rope_
from sglang.kernels.ops.layernorm.norm import fused_inplace_qknorm
from sglang.test.ci.ci_register import register_cuda_ci
Expand Down Expand Up @@ -71,6 +72,26 @@ def test_cuda_sequence_is_bit_exact(d, batch, hq, hk, padded, pos_dtype):
assert_same_bits(storage[:, -8:], before[:, -8:])


@pytest.mark.parametrize("interleaved", [False, True])
@pytest.mark.parametrize("batch,hq,hk", [(1, 32, 8), (32, 8, 1), (257, 64, 8)])
def test_bf16_table_matches_triton_mrope_on_decode_positions(
batch, hq, hk, interleaved
):
"""With equal axes, a BF16 table must reproduce the split MRoPE decode path."""
d = 128
_, x, pos, qw, kw, cache = inputs(d, batch, hq, hk, 29)
cache = cache.bfloat16()
expected = x.clone()
q, k, _ = expected.split([hq * d, hk * d, hk * d], dim=-1)
fused_inplace_qknorm(q.view(batch, hq, d), k.view(batch, hk, d), qw, kw, 1e-6)
positions_3d = pos.unsqueeze(0).repeat(3, 1)
triton_mrope_fused(
q, k, cache, positions_3d, [24, 20, 20], d, d, interleaved, False, True, None
)
table_qk_norm_rope_(x, pos, qw, kw, cache, hq, hk, d, 1e-6, match_cuda=True)
assert_same_bits(x, expected)


@pytest.mark.parametrize("d", [64, 128, 256])
@pytest.mark.parametrize("eps", [1e-6, 1e-5])
def test_extreme_values_and_rounding(d, eps):
Expand Down Expand Up @@ -115,7 +136,7 @@ def test_cuda_mode_rejects_invalid_inputs(invalid):
elif invalid == "qkv_stride":
x = x[:, ::2]
elif invalid == "table":
cache = cache.bfloat16()
cache = cache.half()
else:
pos = pos.float()
with pytest.raises(AssertionError):
Expand Down
40 changes: 33 additions & 7 deletions test/registered/unit/models/test_qwen3_qk_norm_rope.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,19 @@ def make_layer(d=128, dtype=torch.bfloat16, **kwargs):
return layer


MROPE = {
"rope_type": "default",
"mrope_section": [24, 20, 20],
"mrope_interleaved": True,
}


def decode_positions(layer, positions):
if layer.table_rope_positions_ndim == 2:
return positions.unsqueeze(0).repeat(3, 1)
return positions


def separate(layer, positions, x, batch):
with patch.object(layer, "use_table_qk_norm_rope", False):
return layer.forward_prepare_native(positions, x, batch)
Expand All @@ -68,6 +81,7 @@ def assert_same_qkv(actual, expected):
assert torch.equal(got.view(torch.int16), ref.view(torch.int16))


@pytest.mark.parametrize("rope_scaling", [None, MROPE])
@pytest.mark.parametrize(
"mode,dtype,should_fuse",
[
Expand All @@ -77,11 +91,11 @@ def assert_same_qkv(actual, expected):
(ForwardMode.DECODE, torch.float16, False),
],
)
def test_model_dispatch(mode, dtype, should_fuse):
layer = make_layer(dtype=dtype)
def test_model_dispatch(rope_scaling, mode, dtype, should_fuse):
layer = make_layer(dtype=dtype, rope_scaling=rope_scaling)
assert layer.use_table_qk_norm_rope
x = torch.randn(4, layer.hidden_size, device="cuda", dtype=dtype)
positions = torch.arange(4, device="cuda") + 17
positions = decode_positions(layer, torch.arange(4, device="cuda") + 17)
batch = SimpleNamespace(forward_mode=mode)
expected = separate(layer, positions, x, batch)
# Only attention itself is stubbed: construction, projections, norm and RoPE are real.
Expand Down Expand Up @@ -126,11 +140,21 @@ def test_constructor_exclusions(runtime, settings, layer_args, deterministic_env
assert not make_layer(**layer_args).use_table_qk_norm_rope


def test_mrope_one_d_positions_stay_separate():
layer = make_layer(rope_scaling=MROPE)
x = torch.randn(4, layer.hidden_size, device="cuda", dtype=torch.bfloat16)
batch = SimpleNamespace(forward_mode=ForwardMode.DECODE)
with patch.object(qwen3, "table_qk_norm_rope_") as fused:
layer.forward_prepare_native(torch.arange(4, device="cuda"), x, batch)
fused.assert_not_called()


@pytest.mark.parametrize("rope_scaling", [None, MROPE])
@pytest.mark.parametrize("batch_size", [1, 4])
def test_changing_input_graph_replay(batch_size):
layer = make_layer()
def test_changing_input_graph_replay(rope_scaling, batch_size):
layer = make_layer(rope_scaling=rope_scaling)
x = torch.randn(batch_size, layer.hidden_size, device="cuda", dtype=torch.bfloat16)
positions = torch.arange(batch_size, device="cuda")
positions = decode_positions(layer, torch.arange(batch_size, device="cuda"))
batch = SimpleNamespace(forward_mode=ForwardMode.DECODE)
layer.forward_prepare_native(positions, x, batch) # JIT warmup before capture.
graph = torch.cuda.CUDAGraph()
Expand All @@ -144,7 +168,9 @@ def test_changing_input_graph_replay(batch_size):
fused.assert_called_once()
for offset in (0, 17, 2048):
x.normal_()
positions.copy_(torch.arange(batch_size, device="cuda") + offset)
positions.copy_(
decode_positions(layer, torch.arange(batch_size, device="cuda") + offset)
)
graph.replay()
assert_same_qkv(actual, separate(layer, positions, x, batch))

Expand Down
Loading