Repository navigation
[ROCm][DSv4.1] Paged MXFP4 sparse indexer on aiter's MQA-logits kernel #58671
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 28 commits
2e31c95
8f6f55c
5ad59b1
7657413
927bd89
6eb0bdd
4601cb0
7325ca8
4bb424a
79c6e33
2a928e2
bcd1c55
0b3a928
1eff579
aa0cb11
19fd66a
3dc2732
167f0f7
313b20e
1df9119
6f49e14
1183fa9
841b30f
15a007f
0b917fd
7109ab6
f116f7b
269987f
7f04590
9cae237
2529e44
ad48afc
26c5201
cd39d3d
4413b79
8fc68e0
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Large diffs are not rendered by default.
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,156 @@ | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # SPDX-FileCopyrightText: Copyright contributors to the vLLM project | ||
| """The ROCm MXFP4 indexer reads the paged cache in place, so each prefill | ||
| chunk is split back into its requests' rows, which one launch finds through | ||
| the chunk's query_start_loc. A chunk is whole requests or one query slice of a | ||
| long request, and both have to come back with the right rows and the right | ||
| block-table rows. | ||
| """ | ||
|
|
||
| import torch | ||
|
|
||
| from vllm.v1.attention.backends.mla.indexer import ( | ||
| DeepseekV32IndexerPrefillChunkMetadata, | ||
| ) | ||
| from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import ( | ||
| native_decode, | ||
| plan_gather_launches, | ||
| plan_prefill_chunks, | ||
| split_prefill_chunks, | ||
| ) | ||
| from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import rocm_mxfp4_consumer_rows | ||
|
|
||
|
|
||
| def _chunk(block_table, token_start, token_end, row_ends): | ||
| ends = torch.tensor(row_ends, dtype=torch.int32) | ||
| return DeepseekV32IndexerPrefillChunkMetadata( | ||
| block_table=block_table, | ||
| cu_seqlen_ks=torch.full_like(ends, 7), | ||
| cu_seqlen_ke=ends + 7, | ||
| cu_seq_lens=torch.zeros(1, dtype=torch.int32), | ||
| token_to_seq=torch.zeros(1, dtype=torch.int32), | ||
| total_seq_lens=1, | ||
| token_start=token_start, | ||
| token_end=token_end, | ||
| num_reqs=block_table.shape[0], | ||
| ) | ||
|
|
||
|
|
||
| # A decode (1 token), then prefills of 3, 3, 5 and 6 query tokens; the last | ||
| # one is sliced into two chunks. | ||
| QUERY_START_LOC = [0, 1, 4, 7, 12, 18] | ||
| SEQ_LENS = torch.tensor([40, 10, 12, 21, 64]) | ||
| BLOCK_TABLE = torch.arange(20, dtype=torch.int32).view(5, 4) | ||
| CHUNKS = [ | ||
| _chunk(BLOCK_TABLE[1:4], 1, 12, list(range(11))), | ||
| _chunk(BLOCK_TABLE[4:5], 12, 15, [29, 30, 30]), | ||
| _chunk(BLOCK_TABLE[4:5], 15, 18, [31, 31, 32]), | ||
| ] | ||
|
|
||
|
|
||
| def _plans(min_gather_width, block=8): | ||
| return plan_prefill_chunks( | ||
| CHUNKS, | ||
| QUERY_START_LOC, | ||
| SEQ_LENS, | ||
| SEQ_LENS.int() // 2, | ||
| 2, | ||
| block, | ||
| min_gather_width, | ||
| torch.tensor(QUERY_START_LOC, dtype=torch.int32), | ||
| ) | ||
|
|
||
|
|
||
| def test_chunks_split_into_requests(): | ||
| plans = _plans(20.0) | ||
|
|
||
| assert [p.requests for p in plans] == [ | ||
| [(0, 3, 0), (3, 6, 1), (6, 11, 2)], | ||
| [(0, 3, 0)], | ||
| [(0, 3, 0)], | ||
| ] | ||
| # every chunk launches once, with offsets local to the chunk | ||
| assert [p.query_start_loc.tolist() for p in plans] == [ | ||
| [0, 3, 6, 11], | ||
| [0, 3], | ||
| [0, 3], | ||
| ] | ||
| # the logits are as wide as the longest compressed context in the chunk | ||
| assert [p.width for p in plans] == [10, 32, 32] | ||
| assert [p.context_lens.tolist() for p in plans] == [[5, 6, 10], [32], [32]] | ||
| torch.testing.assert_close(plans[2].row_ends, torch.tensor([31, 31, 32]).int()) | ||
| torch.testing.assert_close(plans[2].block_ends, torch.tensor([4, 4, 4]).int()) | ||
| assert [p.use_gather for p in plans] == [False, True, True] | ||
| assert [p.first_request for p in plans] == [1, 4, 4] | ||
| # no candidate blocks, as in the model-neutral builder | ||
| for plan in _plans(None, block=0): | ||
| assert plan.block_ends is None and not plan.use_gather | ||
|
|
||
|
|
||
| def test_launches_follow_the_logits_budget(monkeypatch): | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I dont really understand the purpose of this one? could you please explain it to me here in a comment in simple words?
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I am bypassing K gather for the indexer and reading directly from the paged cache. Since it also comes with its own prefill chunking logic, I am verifying the chunking respects the memory limit (VLLM_SPARSE_INDEXER_MAX_LOGITS_MB) for the output of this kernel in prefill (logits tensor). VLLM_SPARSE_INDEXER_MAX_LOGITS_MB is needed because the output tensor is unbounded (q len x kv len) without external constraint imposed.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. VLLM alreadys uses VLLM_SPARSE_INDEXER_MAX_LOGITS_MB to bound the size of this tensor. |
||
| """Each launch's fp32 logits fit in VLLM_SPARSE_INDEXER_MAX_LOGITS_MB. A dense | ||
| chunk's logits are [rows, longest context in the chunk]: K is read in place, | ||
| not gathered, so requests share a launch until their rows no longer fit, and | ||
| a request that does not fit alone is split on its query rows. A consumer's | ||
| logits are [rows, pool].""" | ||
| budget = 4 * 1000 * 30 | ||
| contexts, queries = torch.tensor([1000, 600, 200]), torch.tensor([10, 10, 10]) | ||
| assert split_prefill_chunks(contexts, queries, budget, 2) == [ | ||
| (slice(2, 5), slice(0, 30)) | ||
| ] | ||
| # the third request would take the chunk past the budget | ||
| queries = torch.tensor([10, 10, 11]) | ||
| assert split_prefill_chunks(contexts, queries, budget) == [ | ||
| (slice(0, 2), slice(0, 20)), | ||
| (slice(2, 3), slice(0, 11)), | ||
| ] | ||
| assert split_prefill_chunks(torch.tensor([1000]), torch.tensor([100]), budget) == [ | ||
| (slice(0, 1), slice(lo, min(lo + 30, 100))) for lo in range(0, 100, 30) | ||
| ] | ||
| # the default 512 MiB holds 128 rows of a 1M context, or 8192 of a 16K pool | ||
| assert split_prefill_chunks( | ||
| torch.tensor([1 << 20]), torch.tensor([200]), 512 << 20 | ||
| ) == [(slice(0, 1), slice(0, 128)), (slice(0, 1), slice(128, 200))] | ||
| monkeypatch.setenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512") | ||
| assert rocm_mxfp4_consumer_rows(16384) == 8192 | ||
|
|
||
|
|
||
| def test_gather_launches_rejoin_a_requests_rows(): | ||
| """The consumers' logits are [rows, pool] whatever the context, so a | ||
| request's rows go back together across the chunks that cut them, and a | ||
| launch never mixes requests.""" | ||
| # the first chunk is too narrow to gather, the last request's slices rejoin | ||
| (joined,) = plan_gather_launches(CHUNKS, _plans(20.0), 100) | ||
| assert (joined.token_start, joined.token_end) == (12, 18) | ||
| # every chunk gathers: the first chunk's three requests launch apart | ||
| launches = plan_gather_launches(CHUNKS, _plans(0.0), 100) | ||
| assert [(g.token_start, g.token_end) for g in launches] == [ | ||
| (1, 4), | ||
| (4, 7), | ||
| (7, 12), | ||
| (12, 18), | ||
| ] | ||
| assert plan_gather_launches(CHUNKS, _plans(None), 100) == [] | ||
|
|
||
|
|
||
| def test_decode_launches_native_on_uniform_steps(): | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. do the mechanisms tested in this file have accuracy checks too?/do they need accuracy checks or are they for functionality? i m sorry -- this is a giant pr so i need your help understanding it
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Basically, this acts as a gate. If every request in decode has same # spec decode entries, the kernel walks the spec decode tokens together so we dont have to read the same KV multiple times. For ragged ones, it just treats them as separate decode tokens, which come at a penalty. With enable_adaptive_verification:false, the alignment should hold and decode attention comes closer to compute bound regime. I think better solution is to always align to next_n tokens even for ragged spec. decode batches but I didnt do that for now.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. test_decode_layers_match_reference should handle the numerical tests for this, for various situations (uniform batches, ragged ones and padded ones). |
||
| """A decode step goes to the kernel as next_n-row sequences only when its | ||
| shape says so: each request has next_n rows, and cudagraph padding (query | ||
| length 0) comes only after them. A FULL graph captured on a uniform batch | ||
| then replays the same launch on a padded one.""" | ||
| rows = torch.zeros(8, dtype=torch.int32) | ||
| row_block_table = torch.arange(32, dtype=torch.int32).view(8, 4) | ||
| context_lens = torch.tensor([9, 4, 7, 0, 5], dtype=torch.int32) | ||
|
|
||
| native = native_decode(rows, row_block_table, [2, 2, 2, 0], 2, context_lens) | ||
| assert native is not None and native.next_n == 2 | ||
| assert native.context_lens.tolist() == [9, 4, 7, 0] | ||
| # every request's first flattened row, as a view | ||
| assert native.block_table.tolist() == row_block_table[::2].tolist() | ||
| assert native.block_table.data_ptr() == row_block_table.data_ptr() | ||
|
|
||
| assert native_decode(rows, row_block_table, [2, 0, 2, 2], 2, context_lens) is None | ||
| # a ragged step, rows padded past the requests, and no speculation | ||
| assert native_decode(rows[:5], row_block_table, [2, 1, 2], 2, context_lens) is None | ||
| assert native_decode(rows, row_block_table, [2, 2, 2], 2, context_lens) is None | ||
| assert native_decode(rows[:4], row_block_table, [1] * 4, 1, context_lens) is None | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| # SPDX-License-Identifier: Apache-2.0 | ||
| # SPDX-FileCopyrightText: Copyright contributors to the vLLM project | ||
| """Sparse attention indexer layers on aiter's paged MXFP4 kernels, for ROCm. | ||
|
|
||
| With ``indexer_kv_dtype="mxfp4"`` on gfx950, the model writes the indexer K | ||
| cache in the order aiter's MQA-logits kernel reads (see | ||
| `rocm_paged_mxfp4_cache_layout`), and the kernel scores the cache in place. | ||
| Requires `RocmMxfp4IndexerMetadataBuilder` metadata. | ||
| """ | ||
|
|
||
| import torch | ||
| from torch import nn | ||
|
|
||
| from vllm.config import get_current_vllm_config | ||
| from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer | ||
| from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import ( | ||
| rocm_mxfp4_sparse_attn_indexer, | ||
| rocm_mxfp4_sparse_mqa_indexer, | ||
| ) | ||
|
|
||
|
|
||
| class RocmSparseAttnIndexer(SparseAttnIndexer): | ||
| """`SparseAttnIndexer` whose HIP path walks the paged MXFP4 cache with | ||
| aiter's kernel. A two-level indexer's source layer also publishes its | ||
| candidate pool here.""" | ||
|
|
||
| def forward_hip( | ||
| self, | ||
| hidden_states: torch.Tensor, | ||
| q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor], | ||
| k: torch.Tensor | None, | ||
| weights: torch.Tensor, | ||
| ): | ||
| assert isinstance(q_quant, tuple) and self.skip_k_cache_insert, ( | ||
| "the ROCm MXFP4 indexer takes (values, scales) Q and a K cache " | ||
| "the model already wrote" | ||
| ) | ||
| q_values, q_scale = q_quant | ||
| return rocm_mxfp4_sparse_attn_indexer( | ||
| hidden_states, | ||
| self.k_cache.prefix, | ||
| self.k_cache.kv_cache, | ||
| q_values, | ||
| q_scale, | ||
| weights, | ||
| self.topk_tokens, | ||
| self.head_dim, | ||
| self.max_model_len, | ||
| self.topk_indices_buffer, | ||
| compress_ratio=self.compress_ratio, | ||
| candidate_blocks=self.candidate_blocks, | ||
| candidate_block_size=self.candidate_block_size, | ||
| candidate_write=self.candidate_write, | ||
| ) | ||
|
|
||
|
|
||
| class RocmSparseMQAIndexer(nn.Module): | ||
| """DeepSeek-V4.1's candidate-consuming indexer on aiter's paged MXFP4 | ||
| MQA-logits kernel: the consumers score only the candidate pool the source | ||
| layer published, read straight from the paged cache. | ||
|
|
||
| Built and called as `SparseMQAIndexer` is, so the model can take either. | ||
| """ | ||
|
|
||
| weights_dtype = torch.float32 | ||
| """The scorer's weights and logits stay fp32: gfx950 has no packed bf16 | ||
| arithmetic, so its head reduce costs the same in either dtype.""" | ||
|
|
||
| def __init__( | ||
| self, | ||
| k_cache, | ||
| topk_tokens: int, | ||
| head_dim: int, | ||
| max_total_seq_len: int, | ||
| topk_indices_buffer: torch.Tensor, | ||
| candidate_blocks: torch.Tensor, | ||
| candidate_block_size: int, | ||
| ): | ||
| super().__init__() | ||
| self.k_cache = k_cache | ||
| self.topk_tokens = topk_tokens | ||
| self.head_dim = head_dim | ||
| # aiter's kernel bounds a row by the model length in compressed | ||
| # positions, not by the prefill buffer SparseMQAIndexer sizes. | ||
| model_config = get_current_vllm_config().model_config | ||
| self.max_model_len = model_config.max_model_len // k_cache.compress_ratio | ||
| self.topk_indices_buffer = topk_indices_buffer | ||
| self.candidate_blocks = candidate_blocks | ||
| self.candidate_block_size = candidate_block_size | ||
| self.num_candidate_cols = candidate_blocks.shape[1] * candidate_block_size | ||
|
|
||
| def forward( | ||
| self, | ||
| hidden_states: torch.Tensor, | ||
| q_quant: tuple[torch.Tensor, torch.Tensor], | ||
| k: torch.Tensor | None, | ||
| weights: torch.Tensor, | ||
| ) -> torch.Tensor: | ||
| assert k is None, "the model writes the indexer K cache" | ||
| q_values, q_scale = q_quant | ||
| return rocm_mxfp4_sparse_mqa_indexer( | ||
| hidden_states, | ||
| self.k_cache.prefix, | ||
| self.k_cache.kv_cache, | ||
| q_values, | ||
| q_scale, | ||
| weights, | ||
| self.topk_tokens, | ||
| self.head_dim, | ||
| self.max_model_len, | ||
| self.topk_indices_buffer, | ||
| self.k_cache.compress_ratio, | ||
| self.candidate_blocks, | ||
| self.candidate_block_size, | ||
| self.num_candidate_cols, | ||
| ) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
this one does not have the skip markers on top like the other test file.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This one is arch agnostic prefill building/chunking logic