Skip to content
Merged
Show file tree
Hide file tree
Changes from 28 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
2e31c95
[ROCm][DSv4.1] Paged MXFP4 sparse indexer on aiter's Gluon MQA-logits…
cagrikymk Sep 23, 2026
8f6f55c
[ROCm][DSv4.1] Prefer 128-token KV pages with the paged MXFP4 indexer
cagrikymk Sep 23, 2026
5ad59b1
[ROCm][DSv4.1] Build the MXFP4 indexer's decode schedule once per step
cagrikymk Sep 23, 2026
7657413
[ROCm][DSv4.1] Launch uniform spec-decode steps as next_n-row sequences
cagrikymk Sep 23, 2026
927bd89
[ROCm][DSv4.1] Let the MXFP4 indexer's consumers gather at any pool size
cagrikymk Sep 23, 2026
6eb0bdd
[ROCm][DSv4.1] Move the MXFP4 indexer to aiter's row_ends API
cagrikymk Sep 23, 2026
4601cb0
[ROCm][DSv4.1] Launch a ragged MXFP4 indexer prefill chunk once, packed
cagrikymk Sep 23, 2026
7325ca8
[ROCm][DSv4.1] Keep the MXFP4 indexer test's page size out of its poo…
cagrikymk Sep 23, 2026
4bb424a
[ROCm][DSv4.1] Name the MXFP4 indexer modules for the paged kernel
cagrikymk Sep 23, 2026
79c6e33
[ROCm][DSv4.1] Read the MXFP4 indexer K page layout from aiter once
cagrikymk Sep 23, 2026
2a928e2
[ROCm][DSv4.1] Resolve the MXFP4 indexer K layout when its cache is b…
cagrikymk Sep 23, 2026
bcd1c55
[ROCm][DSv4.1] Size MXFP4 indexer prefill launches by what each one h…
cagrikymk Sep 24, 2026
0b3a928
[ROCm][DSv4.1] Launch every MXFP4 indexer prefill chunk packed
cagrikymk Sep 24, 2026
1eff579
[ROCm][DSv4.1] Write the MXFP4 indexer K and Q through aiter's cache-…
cagrikymk Sep 24, 2026
aa0cb11
Merge branch 'vllm-project:main' into cagri/rocm-mxfp4-indexer-aiter-…
cagrikymk Sep 25, 2026
19fd66a
[ROCm][DSv4.1] Call aiter's renamed MXFP4 indexer cache ops
cagrikymk Sep 25, 2026
3dc2732
[ROCm][DSv4.1] Resolve the candidate pool once per prefill launch
cagrikymk Sep 25, 2026
167f0f7
[ROCm][DSv4.1] Stop checking the MXFP4 K cache against aiter's refere…
cagrikymk Sep 25, 2026
313b20e
[ROCm][DSv4.1] Reserve 8 B per candidate block for aiter's slot lists
cagrikymk Sep 25, 2026
1df9119
[ROCm][DSv4.1] Import aiter's MXFP4 indexer ops like the rest of vLLM
cagrikymk Sep 25, 2026
6f49e14
[ROCm][DSv4.1] Match get_supported_kernel_block_sizes to its base sig…
cagrikymk Sep 25, 2026
1183fa9
[ROCm][DSv4.1] Use aiter's top-k in the MXFP4 indexer from 128 rows
cagrikymk Sep 25, 2026
841b30f
[ROCm][DSv4.1] Hold the MXFP4 candidate list reservation for the forward
cagrikymk Sep 25, 2026
15a007f
[ROCm][DSv4.1] Scope the MXFP4 indexer to DSv4.1 and keep its paged c…
cagrikymk Sep 28, 2026
0b917fd
Merge branch 'vllm-project:main' into cagri/rocm-mxfp4-indexer-aiter-…
cagrikymk Sep 28, 2026
7109ab6
[ROCm][DSv4.1] Hold the MXFP4 indexer test to a torch MXFP4 reference
cagrikymk Sep 28, 2026
f116f7b
[ROCm][DSv4.1] Fix mypy errors in the MXFP4 indexer path
cagrikymk Sep 28, 2026
269987f
[ROCm][DSv4.1] Size MXFP4 indexer launches by the logits budget alone…
cagrikymk Sep 28, 2026
7f04590
Merge branch 'main' into cagri/rocm-mxfp4-indexer-aiter-cache-ops
cagrikymk Sep 29, 2026
9cae237
[ROCm][DSv4.1] Remove the unused 128-token block size preference
cagrikymk Sep 29, 2026
2529e44
[ROCm][DSv4.1] Rename einsum subscripts that typos reads as a misspel…
cagrikymk Sep 29, 2026
ad48afc
[ROCm][DSv4.1] Gate the MXFP4 indexer on CDNA4 and skip its plan test…
cagrikymk Sep 29, 2026
26c5201
Merge branch 'main' into cagri/rocm-mxfp4-indexer-aiter-cache-ops
cagrikymk Sep 29, 2026
cd39d3d
Merge branch 'main' into cagri/rocm-mxfp4-indexer-aiter-cache-ops
shen-shanshan Sep 29, 2026
4413b79
[ROCm][DSv4.1] Pick the MXFP4 indexer through a ROCm subclass
cagrikymk Sep 29, 2026
8fc68e0
Merge branch 'main' into cagri/rocm-mxfp4-indexer-aiter-cache-ops
cagrikymk Sep 29, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
599 changes: 599 additions & 0 deletions tests/kernels/attention/test_rocm_paged_mxfp4_indexer.py

Large diffs are not rendered by default.

156 changes: 156 additions & 0 deletions tests/v1/attention/test_rocm_paged_mxfp4_indexer_plan.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The ROCm MXFP4 indexer reads the paged cache in place, so each prefill
chunk is split back into its requests' rows, which one launch finds through
the chunk's query_start_loc. A chunk is whole requests or one query slice of a
long request, and both have to come back with the right rows and the right
block-table rows.
"""

import torch

from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerPrefillChunkMetadata,
)
from vllm.v1.attention.backends.mla.rocm_paged_mxfp4_indexer import (
native_decode,
plan_gather_launches,
plan_prefill_chunks,
split_prefill_chunks,
)
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import rocm_mxfp4_consumer_rows

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this one does not have the skip markers on top like the other test file.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one is arch agnostic prefill building/chunking logic


def _chunk(block_table, token_start, token_end, row_ends):
ends = torch.tensor(row_ends, dtype=torch.int32)
return DeepseekV32IndexerPrefillChunkMetadata(
block_table=block_table,
cu_seqlen_ks=torch.full_like(ends, 7),
cu_seqlen_ke=ends + 7,
cu_seq_lens=torch.zeros(1, dtype=torch.int32),
token_to_seq=torch.zeros(1, dtype=torch.int32),
total_seq_lens=1,
token_start=token_start,
token_end=token_end,
num_reqs=block_table.shape[0],
)


# A decode (1 token), then prefills of 3, 3, 5 and 6 query tokens; the last
# one is sliced into two chunks.
QUERY_START_LOC = [0, 1, 4, 7, 12, 18]
SEQ_LENS = torch.tensor([40, 10, 12, 21, 64])
BLOCK_TABLE = torch.arange(20, dtype=torch.int32).view(5, 4)
CHUNKS = [
_chunk(BLOCK_TABLE[1:4], 1, 12, list(range(11))),
_chunk(BLOCK_TABLE[4:5], 12, 15, [29, 30, 30]),
_chunk(BLOCK_TABLE[4:5], 15, 18, [31, 31, 32]),
]


def _plans(min_gather_width, block=8):
return plan_prefill_chunks(
CHUNKS,
QUERY_START_LOC,
SEQ_LENS,
SEQ_LENS.int() // 2,
2,
block,
min_gather_width,
torch.tensor(QUERY_START_LOC, dtype=torch.int32),
)


def test_chunks_split_into_requests():
plans = _plans(20.0)

assert [p.requests for p in plans] == [
[(0, 3, 0), (3, 6, 1), (6, 11, 2)],
[(0, 3, 0)],
[(0, 3, 0)],
]
# every chunk launches once, with offsets local to the chunk
assert [p.query_start_loc.tolist() for p in plans] == [
[0, 3, 6, 11],
[0, 3],
[0, 3],
]
# the logits are as wide as the longest compressed context in the chunk
assert [p.width for p in plans] == [10, 32, 32]
assert [p.context_lens.tolist() for p in plans] == [[5, 6, 10], [32], [32]]
torch.testing.assert_close(plans[2].row_ends, torch.tensor([31, 31, 32]).int())
torch.testing.assert_close(plans[2].block_ends, torch.tensor([4, 4, 4]).int())
assert [p.use_gather for p in plans] == [False, True, True]
assert [p.first_request for p in plans] == [1, 4, 4]
# no candidate blocks, as in the model-neutral builder
for plan in _plans(None, block=0):
assert plan.block_ends is None and not plan.use_gather


def test_launches_follow_the_logits_budget(monkeypatch):

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I dont really understand the purpose of this one? could you please explain it to me here in a comment in simple words?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am bypassing K gather for the indexer and reading directly from the paged cache. Since it also comes with its own prefill chunking logic, I am verifying the chunking respects the memory limit (VLLM_SPARSE_INDEXER_MAX_LOGITS_MB) for the output of this kernel in prefill (logits tensor).

VLLM_SPARSE_INDEXER_MAX_LOGITS_MB is needed because the output tensor is unbounded (q len x kv len) without external constraint imposed.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

VLLM alreadys uses VLLM_SPARSE_INDEXER_MAX_LOGITS_MB to bound the size of this tensor.

"""Each launch's fp32 logits fit in VLLM_SPARSE_INDEXER_MAX_LOGITS_MB. A dense
chunk's logits are [rows, longest context in the chunk]: K is read in place,
not gathered, so requests share a launch until their rows no longer fit, and
a request that does not fit alone is split on its query rows. A consumer's
logits are [rows, pool]."""
budget = 4 * 1000 * 30
contexts, queries = torch.tensor([1000, 600, 200]), torch.tensor([10, 10, 10])
assert split_prefill_chunks(contexts, queries, budget, 2) == [
(slice(2, 5), slice(0, 30))
]
# the third request would take the chunk past the budget
queries = torch.tensor([10, 10, 11])
assert split_prefill_chunks(contexts, queries, budget) == [
(slice(0, 2), slice(0, 20)),
(slice(2, 3), slice(0, 11)),
]
assert split_prefill_chunks(torch.tensor([1000]), torch.tensor([100]), budget) == [
(slice(0, 1), slice(lo, min(lo + 30, 100))) for lo in range(0, 100, 30)
]
# the default 512 MiB holds 128 rows of a 1M context, or 8192 of a 16K pool
assert split_prefill_chunks(
torch.tensor([1 << 20]), torch.tensor([200]), 512 << 20
) == [(slice(0, 1), slice(0, 128)), (slice(0, 1), slice(128, 200))]
monkeypatch.setenv("VLLM_SPARSE_INDEXER_MAX_LOGITS_MB", "512")
assert rocm_mxfp4_consumer_rows(16384) == 8192


def test_gather_launches_rejoin_a_requests_rows():
"""The consumers' logits are [rows, pool] whatever the context, so a
request's rows go back together across the chunks that cut them, and a
launch never mixes requests."""
# the first chunk is too narrow to gather, the last request's slices rejoin
(joined,) = plan_gather_launches(CHUNKS, _plans(20.0), 100)
assert (joined.token_start, joined.token_end) == (12, 18)
# every chunk gathers: the first chunk's three requests launch apart
launches = plan_gather_launches(CHUNKS, _plans(0.0), 100)
assert [(g.token_start, g.token_end) for g in launches] == [
(1, 4),
(4, 7),
(7, 12),
(12, 18),
]
assert plan_gather_launches(CHUNKS, _plans(None), 100) == []


def test_decode_launches_native_on_uniform_steps():

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do the mechanisms tested in this file have accuracy checks too?/do they need accuracy checks or are they for functionality? i m sorry -- this is a giant pr so i need your help understanding it

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Basically, this acts as a gate. If every request in decode has same # spec decode entries, the kernel walks the spec decode tokens together so we dont have to read the same KV multiple times.

For ragged ones, it just treats them as separate decode tokens, which come at a penalty.

With enable_adaptive_verification:false, the alignment should hold and decode attention comes closer to compute bound regime.

I think better solution is to always align to next_n tokens even for ragged spec. decode batches but I didnt do that for now.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

test_decode_layers_match_reference should handle the numerical tests for this, for various situations (uniform batches, ragged ones and padded ones).

"""A decode step goes to the kernel as next_n-row sequences only when its
shape says so: each request has next_n rows, and cudagraph padding (query
length 0) comes only after them. A FULL graph captured on a uniform batch
then replays the same launch on a padded one."""
rows = torch.zeros(8, dtype=torch.int32)
row_block_table = torch.arange(32, dtype=torch.int32).view(8, 4)
context_lens = torch.tensor([9, 4, 7, 0, 5], dtype=torch.int32)

native = native_decode(rows, row_block_table, [2, 2, 2, 0], 2, context_lens)
assert native is not None and native.next_n == 2
assert native.context_lens.tolist() == [9, 4, 7, 0]
# every request's first flattened row, as a view
assert native.block_table.tolist() == row_block_table[::2].tolist()
assert native.block_table.data_ptr() == row_block_table.data_ptr()

assert native_decode(rows, row_block_table, [2, 0, 2, 2], 2, context_lens) is None
# a ragged step, rows padded past the requests, and no speculation
assert native_decode(rows[:5], row_block_table, [2, 1, 2], 2, context_lens) is None
assert native_decode(rows, row_block_table, [2, 2, 2], 2, context_lens) is None
assert native_decode(rows[:4], row_block_table, [1] * 4, 1, context_lens) is None
5 changes: 4 additions & 1 deletion vllm/config/attention.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,10 @@ class AttentionConfig:
an SM100-class GPU, DeepGEMM >= 2.8 and the DeepSelect top-k extension
(the top-k runs on the kernels' bf16 logits). The sparse path costs
O(candidate blocks) per query regardless of context length, so it pays off
for long contexts (roughly 32K tokens and beyond) and is slower below."""
for long contexts (roughly 32K tokens and beyond) and is slower below.
On ROCm gfx950 it runs aiter's paged MXFP4 MQA-logits kernel over the
candidate pool instead, with fp32 logits, and keeps the masked dense walk
for steps whose contexts are too short for the pool to pay."""

hisparse_config: HiSparseConfig | None = None
"""HiSparse host-resident KV configuration. Setting this enables experimental
Expand Down
116 changes: 116 additions & 0 deletions vllm/model_executor/layers/rocm_paged_mxfp4_indexer.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Sparse attention indexer layers on aiter's paged MXFP4 kernels, for ROCm.

With ``indexer_kv_dtype="mxfp4"`` on gfx950, the model writes the indexer K
cache in the order aiter's MQA-logits kernel reads (see
`rocm_paged_mxfp4_cache_layout`), and the kernel scores the cache in place.
Requires `RocmMxfp4IndexerMetadataBuilder` metadata.
"""

import torch
from torch import nn

from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.sparse_attn_indexer import SparseAttnIndexer
from vllm.v1.attention.ops.rocm_paged_mxfp4_indexer import (
rocm_mxfp4_sparse_attn_indexer,
rocm_mxfp4_sparse_mqa_indexer,
)


class RocmSparseAttnIndexer(SparseAttnIndexer):
"""`SparseAttnIndexer` whose HIP path walks the paged MXFP4 cache with
aiter's kernel. A two-level indexer's source layer also publishes its
candidate pool here."""

def forward_hip(
self,
hidden_states: torch.Tensor,
q_quant: torch.Tensor | tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
):
assert isinstance(q_quant, tuple) and self.skip_k_cache_insert, (
"the ROCm MXFP4 indexer takes (values, scales) Q and a K cache "
"the model already wrote"
)
q_values, q_scale = q_quant
return rocm_mxfp4_sparse_attn_indexer(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_values,
q_scale,
weights,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.topk_indices_buffer,
compress_ratio=self.compress_ratio,
candidate_blocks=self.candidate_blocks,
candidate_block_size=self.candidate_block_size,
candidate_write=self.candidate_write,
)


class RocmSparseMQAIndexer(nn.Module):
"""DeepSeek-V4.1's candidate-consuming indexer on aiter's paged MXFP4
MQA-logits kernel: the consumers score only the candidate pool the source
layer published, read straight from the paged cache.

Built and called as `SparseMQAIndexer` is, so the model can take either.
"""

weights_dtype = torch.float32
"""The scorer's weights and logits stay fp32: gfx950 has no packed bf16
arithmetic, so its head reduce costs the same in either dtype."""

def __init__(
self,
k_cache,
topk_tokens: int,
head_dim: int,
max_total_seq_len: int,
topk_indices_buffer: torch.Tensor,
candidate_blocks: torch.Tensor,
candidate_block_size: int,
):
super().__init__()
self.k_cache = k_cache
self.topk_tokens = topk_tokens
self.head_dim = head_dim
# aiter's kernel bounds a row by the model length in compressed
# positions, not by the prefill buffer SparseMQAIndexer sizes.
model_config = get_current_vllm_config().model_config
self.max_model_len = model_config.max_model_len // k_cache.compress_ratio
self.topk_indices_buffer = topk_indices_buffer
self.candidate_blocks = candidate_blocks
self.candidate_block_size = candidate_block_size
self.num_candidate_cols = candidate_blocks.shape[1] * candidate_block_size

def forward(
self,
hidden_states: torch.Tensor,
q_quant: tuple[torch.Tensor, torch.Tensor],
k: torch.Tensor | None,
weights: torch.Tensor,
) -> torch.Tensor:
assert k is None, "the model writes the indexer K cache"
q_values, q_scale = q_quant
return rocm_mxfp4_sparse_mqa_indexer(
hidden_states,
self.k_cache.prefix,
self.k_cache.kv_cache,
q_values,
q_scale,
weights,
self.topk_tokens,
self.head_dim,
self.max_model_len,
self.topk_indices_buffer,
self.k_cache.compress_ratio,
self.candidate_blocks,
self.candidate_block_size,
self.num_candidate_cols,
)
Loading
Loading