Repository navigation
[ROCm][Model][Bugfix] Enable GLM-5.2-MXFP4 on the deepseek_v32 path and fix sparse attention correctness #51915
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 6 commits
73a1c79
12ed6ed
d104154
d91ae1b
70d1891
825a834
e1810f0
df1e2a0
ef4c74f
9e46c41
06b5547
6bdc26e
3ef331a
d4debcb
c2cb9ed
4a9f852
5074aa2
03540d5
1280c56
c3de540
1fdc85a
3f94981
6ac6e08
bad5aa0
d10f12d
6088a13
7835341
a01639d
c58e8dd
b124eff
16bf640
bd3ccea
bf4544e
fcd0680
0e6d4b1
38eecbb
8200224
90b15a9
870eeb3
99226d1
e53ffcb
0906144
2d5c177
c8b6979
998bf01
d93c177
52ab91a
39ba4e5
e8967b3
21135e8
8a1175d
933d45c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,6 +11,12 @@ | |
| # has_indexer=False path so the indexer args don't allocate every call. | ||
| _DUMMY_CACHE: dict[tuple, torch.Tensor] = {} | ||
|
|
||
| # Tile shape of the indexer-K cache's shuffled layout, used when the cache reports | ||
| # uses_shuffled_layout. The block tile is a token count; the head tile is a byte | ||
| # count, which the kernel converts to cache elements. | ||
| _INDEXER_CACHE_BLOCK_TILE = 16 | ||
| _INDEXER_CACHE_HEAD_TILE_BYTES = 16 | ||
|
|
||
|
|
||
| def _dummy(shape: tuple, dtype: torch.dtype, device: torch.device) -> torch.Tensor: | ||
| key = (shape, dtype, device) | ||
|
|
@@ -70,18 +76,30 @@ def _fp8_quant_and_cache_write( | |
| cache_stride, | ||
| offsets, | ||
| HEAD_DIM: tl.constexpr, | ||
| SHUFFLE: tl.constexpr, | ||
| BLOCK_TILE: tl.constexpr, | ||
| HEAD_TILE: tl.constexpr, | ||
| ): | ||
| k_fp8, scale = _fp8_ue8m0_quantize(vals) | ||
|
|
||
| block_idx = slot_idx // cache_block_size | ||
| block_offset = slot_idx % cache_block_size | ||
| block_start = block_idx * cache_block_size * cache_stride | ||
|
|
||
| tl.store( | ||
| kv_cache_ptr + block_start + block_offset * HEAD_DIM + offsets, | ||
| k_fp8, | ||
| mask=mask, | ||
| ) | ||
| # Shuffled layout: [blk/BLOCK_TILE, HEAD_DIM/HEAD_TILE, BLOCK_TILE, HEAD_TILE], | ||
| # so one contiguous run holds BLOCK_TILE tokens x HEAD_TILE bytes and the | ||
| # reader's coalesced loads land on the bytes we wrote. | ||
| if SHUFFLE: | ||
| value_off = ( | ||
| block_offset // BLOCK_TILE * BLOCK_TILE * HEAD_DIM | ||
| + block_offset % BLOCK_TILE * HEAD_TILE | ||
| + offsets // HEAD_TILE * BLOCK_TILE * HEAD_TILE | ||
| + offsets % HEAD_TILE | ||
| ) | ||
| else: | ||
| value_off = block_offset * HEAD_DIM + offsets | ||
|
|
||
| tl.store(kv_cache_ptr + block_start + value_off, k_fp8, mask=mask) | ||
| scale_byte_off = block_start + cache_block_size * HEAD_DIM + block_offset * 4 | ||
| tl.store(kv_cache_scale_ptr + scale_byte_off // 4, scale) | ||
|
|
||
|
|
@@ -129,6 +147,9 @@ def _fused_norm_rope_kernel( | |
| indexer_cache_scale_ptr, | ||
| indexer_cache_block_size, | ||
| indexer_cache_stride, | ||
| INDEXER_CACHE_SHUFFLE: tl.constexpr, | ||
| INDEXER_CACHE_BLOCK_TILE: tl.constexpr, | ||
| INDEXER_CACHE_HEAD_TILE: tl.constexpr, | ||
| # MLA KV cache (concat kv_c_normed + k_pe_roped, uses slot_mapping_ptr) | ||
| mla_cache_ptr, | ||
| mla_cache_block_stride, | ||
|
|
@@ -279,7 +300,7 @@ def _fused_norm_rope_kernel( | |
| # Shared layer: no indexer K to process. | ||
| return | ||
| # Fused: Index K LayerNorm + RoPE + FP8 quant + cache write. | ||
| # Eliminates the separate indexer_k_quant_and_cache kernel launch. | ||
| # Eliminates the separate indexer-K quant-and-cache kernel launch. | ||
|
jhu960213 marked this conversation as resolved.
Resolved
|
||
|
|
||
| index_k_block = tl.arange(0, INDEX_K_BLOCK_SIZE) | ||
| index_k_mask = index_k_block < INDEX_K_DIM | ||
|
|
@@ -367,6 +388,9 @@ def _fused_norm_rope_kernel( | |
| indexer_cache_stride, | ||
| index_k_block, | ||
| INDEX_K_DIM, | ||
| INDEXER_CACHE_SHUFFLE, | ||
| INDEXER_CACHE_BLOCK_TILE, | ||
| INDEXER_CACHE_HEAD_TILE, | ||
| ) | ||
|
|
||
|
|
||
|
|
@@ -389,6 +413,7 @@ def fused_norm_rope( | |
| # Cache params for fused writes (single slot_mapping for both caches) | ||
| slot_mapping: torch.Tensor | None = None, | ||
| indexer_k_cache: torch.Tensor | None = None, | ||
| indexer_cache_shuffled: bool = False, | ||
| mla_kv_cache: torch.Tensor | None = None, | ||
| mla_kv_cache_dtype: str = "auto", | ||
| mla_k_scale: torch.Tensor | None = None, | ||
|
|
@@ -427,6 +452,14 @@ def fused_norm_rope( | |
| idx_cache_scale_view = indexer_k_cache.view(torch.uint8).view(torch.float32) | ||
| idx_cache_block_size = indexer_k_cache.shape[1] | ||
| idx_cache_stride = indexer_k_cache.shape[2] | ||
| # The caller's cache reports whether its reader expects the shuffled | ||
| # layout; see DeepseekV32IndexerCache.uses_shuffled_layout. | ||
| idx_cache_shuffle = indexer_cache_shuffled | ||
| if idx_cache_shuffle: | ||
| assert idx_cache_block_size % _INDEXER_CACHE_BLOCK_TILE == 0, ( | ||
| f"indexer K cache block size {idx_cache_block_size} must be a " | ||
| f"multiple of {_INDEXER_CACHE_BLOCK_TILE} for the shuffled layout" | ||
| ) | ||
| if indexer_k_cache.dtype == torch.uint8: | ||
| indexer_k_cache = indexer_k_cache.view(torch.float8_e4m3fn) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not necessarily related to your change, but should this be
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This line actually predates my changes. I was thinking the same, tbh, I think it's better with |
||
| else: | ||
|
|
@@ -436,12 +469,19 @@ def fused_norm_rope( | |
| indexer_k_cache = torch.empty(0, dtype=torch.float8_e4m3fn, device=device) | ||
| idx_cache_block_size = 1 | ||
| idx_cache_stride = 1 | ||
| idx_cache_shuffle = False | ||
| if mla_kv_cache is None: | ||
| # Pure profiling run (no caches at all): skip all per-token writes. | ||
| slot_mapping = torch.full( | ||
| (num_tokens,), -1, dtype=torch.int64, device=device | ||
| ) | ||
|
|
||
| # The head tile is a byte count and the kernel indexes indexer_k_cache in | ||
| # elements, so convert using the dtype the kernel actually receives. | ||
| idx_cache_head_tile = ( | ||
| _INDEXER_CACHE_HEAD_TILE_BYTES // indexer_k_cache.element_size() | ||
| ) | ||
|
|
||
| # --- MLA KV cache setup --- | ||
| mla_cache_ds_mla = mla_kv_cache_dtype == "fp8_ds_mla" | ||
| mla_cache_fp8 = mla_kv_cache_dtype not in ("auto", "fp8_ds_mla") | ||
|
|
@@ -479,6 +519,9 @@ def fused_norm_rope( | |
| if q_c_out is None: | ||
| q_c_out = torch.empty_like(q_c) | ||
| use_pdl = current_platform.is_arch_support_pdl() | ||
| # `launch_pdl` is a CUDA-only Triton runtime kwarg; ROCm's Triton rejects it | ||
|
jhu960213 marked this conversation as resolved.
Resolved
|
||
| # even when False, so only pass it when PDL is actually supported. | ||
| pdl_kwargs = {"launch_pdl": True} if use_pdl else {} | ||
| _fused_norm_rope_kernel[(4, num_tokens)]( | ||
| positions, | ||
| # Q RMS norm | ||
|
|
@@ -519,6 +562,9 @@ def fused_norm_rope( | |
| idx_cache_scale_view, | ||
| idx_cache_block_size, | ||
| idx_cache_stride, | ||
| idx_cache_shuffle, | ||
| _INDEXER_CACHE_BLOCK_TILE, | ||
| idx_cache_head_tile, | ||
| # MLA KV cache (uses same slot_mapping) | ||
| mla_kv_cache, | ||
| mla_block_stride, | ||
|
|
@@ -538,7 +584,7 @@ def fused_norm_rope( | |
| HAS_INDEXER=has_indexer, | ||
| INDEX_ROPE_INTERLEAVE=index_rope_interleave, | ||
| USE_PDL=use_pdl, | ||
| launch_pdl=use_pdl, | ||
| **pdl_kwargs, | ||
| ) | ||
| return q_c_out | ||
|
|
||
|
|
@@ -869,6 +915,7 @@ def fused_q( | |
| return index_q_fp8, index_weights_out, mqa_q | ||
|
|
||
| use_pdl = current_platform.is_arch_support_pdl() | ||
| pdl_kwargs = {"launch_pdl": True} if use_pdl else {} | ||
| _fused_q_kernel[(3, num_tokens, grid_heads)]( | ||
| positions, | ||
| q_pe, | ||
|
|
@@ -911,7 +958,7 @@ def fused_q( | |
| INDEX_ROPE_INTERLEAVE=index_rope_interleave, | ||
| QUANTIZE_MQA=quantize_mqa, | ||
| USE_PDL=use_pdl, | ||
| launch_pdl=use_pdl, | ||
| **pdl_kwargs, | ||
| # num_warps=1 is optimal here: each program is a single 128-element | ||
| # rope+quant, so the kernel is program-count/occupancy bound, not | ||
| # per-program compute bound (swept 1/2/4/8 — 1 wins or ties everywhere). | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.