Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
2721e26
[Attention] Add masked MHA path for sparse MLA
MatthewBonanni Jul 15, 2026
e2b53b6
Fix IMA
MatthewBonanni Jul 15, 2026
3008fcc
Clean up padding
MatthewBonanni Jul 15, 2026
b47873f
Merge branch 'main' into fa4_masked_mha_remainder
MatthewBonanni Jul 16, 2026
15d6999
Chunked context support
MatthewBonanni Jul 16, 2026
a3ca0e2
Reuse global topk mask
MatthewBonanni Jul 16, 2026
1a37b3c
Remove block sparse stuff
MatthewBonanni Jul 16, 2026
c19aa02
Add heuristic
MatthewBonanni Jul 16, 2026
81ec3e4
Merge branch 'main' into fa4_masked_mha_remainder
MatthewBonanni Jul 17, 2026
08524a0
Merge branch 'main' into fa4_masked_mha_remainder
MatthewBonanni Jul 20, 2026
6562310
Merge branch 'main' into fa4_masked_mha_remainder
MatthewBonanni Jul 23, 2026
512b29b
Merge branch 'main' into fa4_masked_mha_remainder
MatthewBonanni Jul 27, 2026
cfc2957
Review comment
MatthewBonanni Jul 27, 2026
25e0528
Cleanup
MatthewBonanni Jul 27, 2026
b09ba1d
Cleanup
MatthewBonanni Jul 27, 2026
f17b1a7
Merge branch 'main' into fa4_masked_mha_remainder
MatthewBonanni Jul 28, 2026
7a0c45a
Cleanup
MatthewBonanni Jul 28, 2026
c68f1a1
Cleanup
MatthewBonanni Jul 28, 2026
8e1241a
Cleanup, break out helper
MatthewBonanni Jul 28, 2026
002ed43
Preallocate topk mask buffer
MatthewBonanni Jul 28, 2026
247048e
Gate workspace allocation on masked mha availability
MatthewBonanni Jul 29, 2026
c266472
Clean up gate and heuristic
MatthewBonanni Jul 29, 2026
e8f3507
Fix tests
MatthewBonanni Jul 29, 2026
34a709f
update FA
MatthewBonanni Jul 29, 2026
5722c83
Zero in kernel
MatthewBonanni Jul 29, 2026
8d44a98
update FA
MatthewBonanni Jul 29, 2026
469862e
update FA after landing
MatthewBonanni Jul 30, 2026
6338c21
Merge branch 'main' into fa4_masked_mha_remainder
MatthewBonanni Jul 30, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion benchmarks/attention_benchmarks/benchmark.py
Original file line number Diff line number Diff line change
Expand Up @@ -617,7 +617,7 @@ def main():
"--sparse-mla-mha-variants",
nargs="+",
default=None,
choices=["dense_mha", "mqa"],
choices=["dense_mha", "masked_mha", "mqa"],
help="Sparse MLA variants to run in mha_vs_mqa mode. Defaults to both.",
)

Expand Down Expand Up @@ -748,6 +748,9 @@ def main():
args.sparse_mla_mha_variants = yaml_config.get(
"sparse_mla_mha_variants", args.sparse_mla_mha_variants
)
args.sparse_mla_masked_mha_max_seq_len = yaml_config.get(
"sparse_mla_masked_mha_max_seq_len", None
)

# Parameter sweep configuration
if "parameter_sweep" in yaml_config:
Expand Down Expand Up @@ -1119,6 +1122,7 @@ def main():
console.print(f"Prefill backend: {prefill_backend}")
available_variants = [
("dense_mha", False, "dense"),
("masked_mha", False, "masked"),
("mqa", True, "auto"),
]
requested_variants = getattr(args, "sparse_mla_mha_variants", None)
Expand All @@ -1139,6 +1143,9 @@ def main():
]
else:
variants = available_variants
masked_mha_max_seq_len = getattr(
args, "sparse_mla_masked_mha_max_seq_len", None
)
formatter = ResultsFormatter(console)
total = 0
for spec in args.batch_specs:
Expand All @@ -1148,6 +1155,10 @@ def main():
variant_label == "dense_mha"
and dense_mha_max_seq_len is not None
and q_len > dense_mha_max_seq_len
) or (
variant_label == "masked_mha"
and masked_mha_max_seq_len is not None
and q_len > masked_mha_max_seq_len
):
continue
total += len(backends)
Expand All @@ -1161,6 +1172,10 @@ def main():
variant_label == "dense_mha"
and dense_mha_max_seq_len is not None
and q_len > dense_mha_max_seq_len
) or (
variant_label == "masked_mha"
and masked_mha_max_seq_len is not None
and q_len > masked_mha_max_seq_len
):
continue
config = BenchmarkConfig(
Expand Down Expand Up @@ -1190,6 +1205,7 @@ def main():
sparse_mla_dense_mha_max_seq_len=dense_mha_max_seq_len,
sparse_mla_topk_pattern=sparse_mla_topk_pattern,
prefill_backend=prefill_backend,
sparse_mla_masked_mha_max_seq_len=masked_mha_max_seq_len,
)

# run_mla_benchmark needs the real backend name
Expand Down
3 changes: 2 additions & 1 deletion benchmarks/attention_benchmarks/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,8 +299,9 @@ class BenchmarkConfig:
num_kv_splits: int | None = None # CUTLASS MLA
reorder_batch_threshold: int | None = None # FlashAttn MLA, FlashMLA
sparse_mla_force_mqa: bool = False # Force MQA path for sparse MLA
sparse_mla_mha_mode: str = "auto" # "auto" or "dense"
sparse_mla_mha_mode: str = "auto" # "auto", "dense", or "masked"
sparse_mla_dense_mha_max_seq_len: int | None = None
sparse_mla_masked_mha_max_seq_len: int | None = None
sparse_mla_topk_pattern: str = "random" # "random", "prefix", "sliding_window"
num_splits: int | None = None # FlashAttention split-K (0=auto, 1=disabled)

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
# Sparse MLA benchmark: forward_mha vs forward_mqa crossover plot
#
# Dense pure-prefill sweep to generate a smooth speedup curve. The 100 points
# run from q32 through q262144, approximately even on the log2 scale, and include
# routing boundaries plus extra samples around the 6k-8k crossover region.
# Dense MHA is only benchmarked through q32768 because it dominates runtime above
# that range.
#
# Usage:
# python benchmark.py --config configs/mla_sparse_mha_vs_mqa.yaml

mode: mha_vs_mqa

model:
name: "deepseek-v3"
num_layers: 60
num_q_heads: 16
num_kv_heads: 1
head_dim: 576
kv_lora_rank: 512
qk_nope_head_dim: 128
qk_rope_head_dim: 64
v_head_dim: 128
block_size: 128
max_model_len: 262144

batch_specs:
- "q32"
- "q36"
- "q40"
- "q45"
- "q50"
- "q56"
- "q62"
- "q70"
- "q78"
- "q87"
- "q97"
- "q109"
- "q122"
- "q136"
- "q152"
- "q170"
- "q190"
- "q212"
- "q237"
- "q255"
- "q256"
- "q257"
- "q265"
- "q296"
- "q331"
- "q370"
- "q413"
- "q462"
- "q516"
- "q577"
- "q645"
- "q721"
- "q806"
- "q901"
- "q1007"
- "q1125"
- "q1257"
- "q1405"
- "q1571"
- "q1756"
- "q1962"
- "q2048"
- "q2049"
- "q2193"
- "q2451"
- "q2740"
- "q3062"
- "q3422"
- "q3825"
- "q4275"
- "q4778"
- "q5340"
- "q5854"
- "q5969"
- "q6000"
- "q6144"
- "q6367"
- "q6671"
- "q6925"
- "q7168"
- "q7456"
- "q7532"
- "q8192"
- "q8193"
- "q8333"
- "q9314"
- "q10410"
- "q11635"
- "q13004"
- "q14534"
- "q16244"
- "q18156"
- "q20292"
- "q22680"
- "q25349"
- "q28332"
- "q31665"
- "q32768"
- "q32769"
- "q35391"
- "q39556"
- "q44210"
- "q49413"
- "q55227"
- "q61726"
- "q65536"

backends:
- FLASHMLA_SPARSE

device: "cuda:0"
profile_memory: true
sparse_mla_dense_mha_max_seq_len: 25349
sparse_mla_masked_mha_max_seq_len: 8192
sparse_mla_mha_variants: ["dense_mha", "masked_mha", "mqa"]
sparse_mla_topk_pattern: "random"
7 changes: 7 additions & 0 deletions benchmarks/attention_benchmarks/mla_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -931,10 +931,17 @@ def _run_single_benchmark(
prefill_quant_op = QuantFP8(static=True, group_shape=GroupShape.PER_TENSOR)

fused_output = output_scale is not None and fuse_quant_op
mha_mode = getattr(config, "sparse_mla_mha_mode", "auto")

# Build forward function (runs a single decode/prefill pass)
def forward_fn():
results = []
impl._sparse_mla_force_masked_mha = ( # type: ignore[attr-defined]
mha_mode == "masked"
)
impl._sparse_mla_force_dense_mha = ( # type: ignore[attr-defined]
mha_mode == "dense"
)
if has_decode:
results.append(impl.forward_mqa(decode_inputs, kv_cache, metadata, layer))
if has_prefill:
Expand Down
4 changes: 2 additions & 2 deletions cmake/external_projects/vllm_flash_attn.cmake
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@ if(VLLM_FLASH_ATTN_SRC_DIR)
else()
FetchContent_Declare(
vllm-flash-attn
GIT_REPOSITORY https://github.com/vllm-project/flash-attention.git
GIT_TAG caaa4eb59845388a20b1f435ecaafb4bd9517ad8
GIT_REPOSITORY https://github.com/MatthewBonanni/flash-attention.git
GIT_TAG 7a2380d5994bfe138e514cfce043a015b735b060
GIT_PROGRESS TRUE
# Don't share the vllm-flash-attn build between build types
BINARY_DIR ${CMAKE_BINARY_DIR}/vllm-flash-attn
Expand Down
34 changes: 28 additions & 6 deletions tests/v1/attention/test_sparse_mla_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -862,6 +862,8 @@ def test_split_prefill_chunks(seq_lens, max_buf, expected):
PREFILL_BATCH_SPECS = {
"short_dense_mha": BatchSpec(seq_lens=[64, 128], query_lens=[64, 128]),
"short_context_dense_mha": BatchSpec(seq_lens=[128, 160], query_lens=[64, 32]),
"masked_mha": BatchSpec(seq_lens=[256], query_lens=[256]),
"masked_mha_chunked_context": BatchSpec(seq_lens=[448, 384], query_lens=[256, 256]),
}


Expand All @@ -878,7 +880,7 @@ def test_sparse_backend_prefill_correctness(
kv_cache_dtype,
workspace_init,
):
"""Test single-pass FA4 dense forward_mha for sparse MLA prefill."""
"""Test single-pass dense and masked MHA for sparse MLA prefill."""
backend_cls = FlashMLASparseBackend
batch_spec = PREFILL_BATCH_SPECS[batch_name]

Expand All @@ -892,7 +894,8 @@ def test_sparse_backend_prefill_correctness(
qk_rope_head_dim = 64
v_head_dim = 128
head_size = kv_lora_rank + qk_rope_head_dim
topk_tokens = 512
masked_mha = batch_name.startswith("masked_mha")
topk_tokens = 200 if masked_mha else 512

max_seqlen = max(batch_spec.seq_lens)
total_cache_tokens = sum(batch_spec.seq_lens)
Expand Down Expand Up @@ -942,13 +945,17 @@ def test_sparse_backend_prefill_correctness(

# Compute dense reference outputs.
total_query_tokens = sum(query_lens)
sparse_indices = torch.zeros(
total_query_tokens, topk_tokens, dtype=torch.int32, device=device
sparse_indices = torch.full(
(total_query_tokens, topk_tokens),
-1 if masked_mha else 0,
Comment thread
MatthewBonanni marked this conversation as resolved.
Resolved
dtype=torch.int32,
device=device,
)

all_q, all_kv_c_new, all_k_pe_new = [], [], []
kv_c_contexts, k_pe_contexts = [], []
reference_outputs = []
global_token_idx = 0

for i in range(batch_spec.batch_size):
s_len = seq_lens[i]
Expand Down Expand Up @@ -981,8 +988,15 @@ def test_sparse_backend_prefill_correctness(
for j in range(q_len):
attend_end = ctx_len + j + 1
q_tok = q_mha[j : j + 1] # (1, H, D_qk)
k_attend = k_all[:attend_end] # (N, H, D_qk)
v_attend = v_all[:attend_end] # (N, H, D_v)
if masked_mha:
actual_topk = min(topk_tokens, attend_end)
attend_indices = torch.randperm(attend_end, device=device)[:actual_topk]
sparse_indices[global_token_idx, :actual_topk] = attend_indices
k_attend = k_all[attend_indices]
v_attend = v_all[attend_indices]
else:
k_attend = k_all[:attend_end] # (N, H, D_qk)
v_attend = v_all[:attend_end] # (N, H, D_v)

q_sdpa = q_tok.unsqueeze(0).transpose(1, 2).float()
k_sdpa = k_attend.unsqueeze(0).transpose(1, 2).float()
Expand All @@ -993,6 +1007,7 @@ def test_sparse_backend_prefill_correctness(
)
out = out.transpose(1, 2).squeeze(0) # (1, H, D_v)
reference_outputs.append(out.to(dtype).flatten(start_dim=-2))
global_token_idx += 1

all_q.append(q_mha)
all_kv_c_new.append(kv_c_full[ctx_len:])
Expand Down Expand Up @@ -1047,6 +1062,13 @@ def test_sparse_backend_prefill_correctness(

builder_cls = backend_cls.get_builder_cls()
builder = builder_cls(kv_cache_spec, ["placeholder"], vllm_config, device)
if batch_name == "masked_mha_chunked_context":
builder.chunked_prefill_workspace_size = block_size * batch_spec.batch_size
builder.chunked_prefill_workspace = torch.empty(
(builder.chunked_prefill_workspace_size, head_size),
dtype=dtype,
device=device,
)
# Drive the queries through the dense-MHA prefill path directly (the routing
# threshold would otherwise classify these short queries as MQA decodes).
builder.reorder_batch_threshold = 1
Expand Down
26 changes: 26 additions & 0 deletions tests/v1/attention/test_sparse_mla_mask.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import pytest
import torch

pytest.importorskip("cutlass")

from vllm.model_executor.layers.attention.sparse_mla_attention import (
_build_topk_mask,
)


@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
def test_build_topk_mask_single_request_matches_generic_path() -> None:
topk = torch.tensor(
[[0, 31, 32, 63, -1], [1, 64, 127, -1, -1]],
dtype=torch.int32,
device="cuda",
)

single_req = _build_topk_mask([topk], [2], 2, 128, topk.device)
generic = _build_topk_mask([topk[:1], topk[1:]], [1, 1], 1, 128, topk.device)

torch.testing.assert_close(single_req[0, 0], generic[0, 0])
torch.testing.assert_close(single_req[0, 1], generic[1, 0])
Loading
Loading