Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions tests/model_executor/layers/test_mla_short_prefill_indexer.py
Original file line number Diff line number Diff line change
Expand Up @@ -378,7 +378,12 @@ def test_select_candidate_blocks_tolerates_empty_rows():
def test_candidate_kernels_preserve_packed_bounds_and_padding(
width, block_size, k, decode
):
"""Preserve top-k ties, newest blocks, empty rows and candidate clamping."""
"""Preserve top-k ties, newest blocks, empty rows and candidate clamping.

The mask covers each row's live ``[0, end)`` span; columns at or past the
row's end keep whatever the logits buffer held, because nothing reads them
(the row top-k takes the same bounds).
"""
from vllm.model_executor.kernels.attention.dsa.candidate_blocks import (
apply_candidate_mask,
select_candidate_blocks,
Expand Down Expand Up @@ -436,7 +441,8 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
reduce="amax",
include_self=True,
)
reference = logits.masked_fill((keep == 0) | ~valid, -torch.inf)
live = cols < ke[:, None]
reference = torch.where(live & ((keep == 0) | ~valid), -torch.inf, logits)
apply_candidate_mask(logits, starts, ends, candidates, block_size, repeat)
torch.testing.assert_close(logits, reference, rtol=0, atol=0, equal_nan=True)
graph = torch.cuda.CUDAGraph()
Expand All @@ -448,7 +454,7 @@ def test_candidate_kernels_preserve_packed_bounds_and_padding(
logits.fill_(3.0)
graph.replay()
keep = valid & (block >= 0) & ((cols - ks[:, None]) // block_size == block)
reference = torch.where(keep, 3.0, -torch.inf)
reference = torch.where(keep | ~live, 3.0, -torch.inf)
torch.testing.assert_close(logits, reference, rtol=0, atol=0)


Expand Down
96 changes: 65 additions & 31 deletions vllm/model_executor/kernels/attention/dsa/candidate_blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,20 @@

from vllm.triton_utils import tl, triton

# Programs along the column axis of the kernels that walk a row's live range.
# The grid is (rows, this) and both terms come from tensor shapes only, so a
# full cudagraph replays the same launch; only the loop trip count inside the
# kernel follows each row's context length. The decode logits buffer is
# max_model_len wide (1M for DeepSeek-V4.1), so a grid sized by the buffer
# width launches 1024 programs per row of which the first few do all the work.
_ROW_PROGRAMS = 8 # floor, so a wide batch keeps a few programs per row
_TOTAL_PROGRAMS = 8192 # ceiling on the total, so a narrow batch still fills


def _row_programs(rows: int, tiles: int) -> int:
"""Column programs per row: enough to fill the device, at most `tiles`."""
return max(1, min(tiles, max(_ROW_PROGRAMS, triton.cdiv(_TOTAL_PROGRAMS, rows))))


@triton.jit
def _max_with_nan(a, b):
Expand All @@ -28,27 +42,39 @@ def _block_scores_kernel(
ROW_REPEAT: tl.constexpr,
TILE: tl.constexpr,
):
# Score only the blocks the row's own [start, end) covers, striding over
# them with a fixed number of programs. `scores` arrives pre-filled with
# -inf, which is what the old full-width grid wrote for the rest.
row = tl.program_id(0).to(tl.int64)
blocks = tl.program_id(1) * TILE + tl.arange(0, TILE)
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
end = tl.load(ends + row // ROW_REPEAT * stride_end)
offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE))
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
values = tl.load(
logits + row * stride_row + cols * stride_col,
(blocks[:, None] < nblocks)
& (offsets[None, :] < BLOCK_SIZE)
& (cols < end)
& (cols < width),
other=-float("inf"),
)
reduced = tl.reduce(values, 1, _max_with_nan)
reduced = tl.where(
(end > start) & (blocks == (end - start - 1) // BLOCK_SIZE),
float("inf"),
reduced,
row_blocks = tl.minimum(
tl.cdiv(tl.maximum(tl.minimum(end, width) - start, 0), BLOCK_SIZE), nblocks
)
tl.store(scores + row * nblocks + blocks, reduced, blocks < nblocks)
# The row's newest block is pinned to +inf, off the unclamped end as before.
pin = (end - start - 1) // BLOCK_SIZE
offsets = tl.arange(0, triton.next_power_of_2(BLOCK_SIZE))
tile = tl.program_id(1)
num_tiles = tl.cdiv(row_blocks, TILE)
while tile < num_tiles:
blocks = tile * TILE + tl.arange(0, TILE)
cols = start + blocks[:, None] * BLOCK_SIZE + offsets[None, :]
values = tl.load(
logits + row * stride_row + cols * stride_col,
(blocks[:, None] < row_blocks)
& (offsets[None, :] < BLOCK_SIZE)
& (cols < end)
& (cols < width),
other=-float("inf"),
)
reduced = tl.reduce(values, 1, _max_with_nan)
reduced = tl.where((end > start) & (blocks == pin), float("inf"), reduced)
tl.store(scores + row * nblocks + blocks, reduced, blocks < row_blocks)
tile += tl.num_programs(1)
# A row whose end runs past the buffer width pins a block outside the range
# walked above; the full-width grid used to reach it.
if (tl.program_id(1) == 0) & (end > start) & (pin >= row_blocks) & (pin < nblocks):
tl.store(scores + row * nblocks + pin, float("inf"))


@triton.jit(do_not_specialize=["k"])
Expand Down Expand Up @@ -121,20 +147,28 @@ def _mask_candidates_kernel(
ROW_REPEAT: tl.constexpr,
TILE: tl.constexpr,
):
# Stride over the row's live [0, end) columns instead of the whole buffer
# width. Columns at or past `end` are not read afterwards -- the row top-k
# bounds its scan by the same ends -- so the buffer tail is left alone.
row = tl.program_id(0).to(tl.int64)
cols = tl.program_id(1) * TILE + tl.arange(0, TILE)
start = tl.load(starts + row // ROW_REPEAT * stride_start) if HAS_STARTS else 0
end = tl.load(ends + row // ROW_REPEAT * stride_end)
valid = (cols >= start) & (cols < end) & (cols < width)
block = (cols - start) // BLOCK_SIZE
keep = tl.load(flags + row * (nblocks + 1) + block, valid, other=0)
edge = tl.load(flags + row * (nblocks + 1) + nblocks)
keep = (keep != 0) | ((cols == width - 1) & (edge != 0))
tl.store(
logits + row * stride_row + cols * stride_col,
-float("inf"),
(cols < width) & ~(valid & keep),
)
col0 = tl.program_id(1) * TILE
step = tl.num_programs(1) * TILE
while col0 < end:
cols = col0 + tl.arange(0, TILE)
live = (cols < end) & (cols < width)
valid = live & (cols >= start)
block = (cols - start) // BLOCK_SIZE
keep = tl.load(flags + row * (nblocks + 1) + block, valid, other=0)
keep = (keep != 0) | ((cols == width - 1) & (edge != 0))
tl.store(
logits + row * stride_row + cols * stride_col,
-float("inf"),
live & ~(valid & keep),
)
col0 += step


def select_candidate_blocks(
Expand All @@ -159,8 +193,8 @@ def select_candidate_blocks(
out.fill_(-1)
return
nblocks = triton.cdiv(width, block_size)
scores = logits.new_empty((rows, nblocks))
_block_scores_kernel[(rows, triton.cdiv(nblocks, 128))](
scores = logits.new_full((rows, nblocks), -float("inf"))
_block_scores_kernel[(rows, _row_programs(rows, triton.cdiv(nblocks, 128)))](
logits,
row_ks,
row_ke,
Expand Down Expand Up @@ -196,7 +230,7 @@ def apply_candidate_mask(
block_size: int,
row_repeat: int = 1,
) -> None:
"""Mask packed logits outside causal bounds and request-local candidates."""
"""Mask each row's live logits down to its request-local candidates."""
assert logits.is_cuda
rows, width = logits.shape
if not rows or not width:
Expand All @@ -217,7 +251,7 @@ def apply_candidate_mask(
row_ks is not None,
row_repeat,
)
_mask_candidates_kernel[(rows, triton.cdiv(width, 1024))](
_mask_candidates_kernel[(rows, _row_programs(rows, triton.cdiv(width, 1024)))](
logits,
row_ks,
row_ke,
Expand Down
Loading