Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 149 additions & 0 deletions tests/v1/attention/test_dcp_a2a_pack_mask.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The empty-shard mask fused into the DCP A2A pack kernel must be bit-exact
with the eager ``mask_dcp_empty_shards_`` pass it replaces."""

import pytest
import torch

from vllm.v1.attention.ops.dcp import (
_dcp_a2a_lse_pack_dim,
_dcp_a2a_pack_send,
mask_dcp_empty_shards_,
)

pytestmark = pytest.mark.skipif(
not torch.cuda.is_available(), reason="needs a GPU for the Triton pack kernel"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wait does this evaluate to true or false for us? I thought we should be using cuda_alike

@rbrugaro-amd rbrugaro-amd Sep 10, 2026 •

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It does evaluate correctly to True, but it is the wrong predicate to express the intent, and I have switched it.

torch.cuda.is_available() returns True on ROCm, because the PyTorch ROCm build maps HIP onto the torch.cuda API. So the file was running on our hardware rather than being skipped. Measured in the ROCm image:

torch.cuda.is_available()        = True
torch.version.hip                = 7.2.53211
current_platform.is_cuda()       = False
current_platform.is_cuda_alike() = True

The trap is the neighbouring predicate: current_platform.is_cuda() is False on ROCm and would have silently skipped the whole file.

Switched to [current_platform.is_cuda_alike()] which is what the rest of the tree uses for "CUDA or ROCm"

No behavioural change; 353 tests still pass on MI355X.

)


def _pack(out, lse, world_size, h_per_rank, head_dim, seq_lens, query_start_loc):
lse_pack_dim = _dcp_a2a_lse_pack_dim(out.dtype)
send = torch.zeros(
(world_size, out.shape[0], h_per_rank, head_dim + lse_pack_dim),
device=out.device,
dtype=out.dtype,
)
_dcp_a2a_pack_send(
out,
lse,
send,
world_size,
h_per_rank,
head_dim,
lse_pack_dim,
seq_lens=seq_lens,
query_start_loc=query_start_loc,
)
return send


@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("world_size,h_per_rank", [(8, 2), (8, 16), (4, 4), (2, 1)])
@pytest.mark.parametrize("tokens_per_req", [1, 2, 3])
@pytest.mark.parametrize("num_pad_rows", [0, 2])
@pytest.mark.parametrize(
"seq_lens_list",
[
[16, 16, 16, 16], # no empty shards (the long-context decode case)
[16, 0, 16, 0], # interior empty shards
[0, 0, 0, 0], # every shard empty
[16, 16, 0, 0], # trailing empty shards, i.e. cudagraph padding
[0, 16, 16, 16], # leading empty shard
[16] * 52, # the profiled conc-52 decode batch
[7], # single request
],
)
def test_fused_mask_matches_eager(
dtype, world_size, h_per_rank, tokens_per_req, num_pad_rows, seq_lens_list
):
torch.manual_seed(0)
device = "cuda"
head_dim = 512
num_seqs = len(seq_lens_list)
# tokens_per_req > 1 is the MTP / multi-token-verify shape.
num_rows = num_seqs * tokens_per_req + num_pad_rows
num_heads = world_size * h_per_rank

out = torch.randn(num_rows, num_heads, head_dim, device=device, dtype=dtype)
lse = torch.randn(num_rows, num_heads, device=device, dtype=torch.float32)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we do we need 32 bit numerics here?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The LSE is fp32 in production — the test is mirroring the real dtype, not choosing one. All of the following predate this PR:

  • _dcp_a2a_lse_pack_dim exists solely to pack an fp32 LSE into a narrower output dtype, and raises "Cannot pack fp32 LSE into output dtype {...}"
  • out_lse is allocated torch.float32
  • the direct symmetric-memory path allocates received_lse as torch.float32

It has to be fp32 for range — LSE is a log of summed exponentials, and this mask writes -inf into it. A 16-bit LSE would lose that.

seq_lens = torch.tensor(seq_lens_list, device=device, dtype=torch.int32)
query_start_loc = torch.arange(
0,
(num_seqs + 1) * tokens_per_req,
tokens_per_req,
device=device,
dtype=torch.int32,
) # rows past query_start_loc[-1] are cudagraph padding

# Reference: mask eagerly, then pack with masking disabled.
ref_lse = lse.clone()
mask_dcp_empty_shards_(ref_lse, seq_lens, query_start_loc)
expected = _pack(out, ref_lse, world_size, h_per_rank, head_dim, None, None)

# Under test: pack with the mask fused in.
actual = _pack(
out, lse.clone(), world_size, h_per_rank, head_dim, seq_lens, query_start_loc
)

# Compare bit patterns, not values: for a 2-element LSE pack the high half
# of -inf bitcasts to a NaN payload, which never compares equal to itself.
assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)), (

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

so is this correct? (i.e. using 32 bit numerics in the beginning and then for the comparison only converting them to 16 bit ones)

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think my comment was misleading, and I have rewritten it in db07f2ab.

Two different tensors are in play. The LSE is fp32 and stays fp32. The send buffer is in the output dtype (16-bit), and _dcp_a2a_lse_pack_dim returns 2 for any 16-bit output, so the fp32 LSE is bit-split across two of its slots. .view(torch.int16) bitcasts the send buffer — reinterpreting 16-bit storage as 16-bit integers. Nothing is narrowed.

Those slots are not meaningful floats at all; they are an opaque container for half of an fp32. A masked row stores -inf = 0xFF800000, and the high half 0xFF80 reads differently depending on the format:

bf16 (1|8|7):   1 | 11111111 | 0000000      exp all-ones, mantissa 0     -> -inf
fp16 (1|5|10):  1 | 11111    | 1110000000   exp all-ones, mantissa 896   -> NaN

bf16 is the top 16 bits of fp32 (same 8-bit exponent), so the truncated half of -inf is still -inf. fp16 has only a 5-bit exponent, so three of fp32's exponent bits spill into fp16's mantissa — and an all-ones exponent with a non-zero mantissa is NaN, which never compares equal to itself.

Bit equality is also strictly stronger than allclose, which is what a bit-exactness claim should be asserting.

"fused mask is not bit-exact with the eager mask"
)


@pytest.mark.parametrize("dtype", [torch.bfloat16, torch.float16])
@pytest.mark.parametrize("seed", range(8))
def test_fused_mask_matches_eager_ragged(dtype, seed):
"""Non-uniform per-request query lengths.

The uniform-stride cases above never exercise an unequal gap between
consecutive ``query_start_loc`` entries, which is what the in-kernel
boundary search has to get right. MTP with partial acceptance produces
exactly this: some requests contribute one row, others several, in the
same batch.
"""
torch.manual_seed(seed)
device = "cuda"
world_size, h_per_rank, head_dim = 8, 2, 512
num_seqs = 17

query_lens = torch.randint(1, 5, (num_seqs,), dtype=torch.int32)
query_start_loc = torch.zeros(num_seqs + 1, dtype=torch.int32)
query_start_loc[1:] = torch.cumsum(query_lens, 0)
# Mix real, empty, and (via padding rows) out-of-range shards.
seq_lens = torch.randint(0, 2, (num_seqs,), dtype=torch.int32) * 16
num_rows = int(query_start_loc[-1]) + 3

query_start_loc = query_start_loc.to(device)
seq_lens = seq_lens.to(device)
num_heads = world_size * h_per_rank
out = torch.randn(num_rows, num_heads, head_dim, device=device, dtype=dtype)
lse = torch.randn(num_rows, num_heads, device=device, dtype=torch.float32)

ref_lse = lse.clone()
mask_dcp_empty_shards_(ref_lse, seq_lens, query_start_loc)
expected = _pack(out, ref_lse, world_size, h_per_rank, head_dim, None, None)
actual = _pack(
out, lse.clone(), world_size, h_per_rank, head_dim, seq_lens, query_start_loc
)

assert torch.equal(actual.view(torch.int16), expected.view(torch.int16)), (
f"ragged query_start_loc mismatch: query_lens={query_lens.tolist()}, "
f"seq_lens={seq_lens.tolist()}"
)


def test_mask_disabled_is_unmasked():
"""Passing no seq_lens/query_start_loc must leave the LSE untouched."""
torch.manual_seed(0)
device = "cuda"
world_size, h_per_rank, head_dim = 8, 2, 512
out = torch.randn(
4, world_size * h_per_rank, head_dim, device=device, dtype=torch.bfloat16
)
lse = torch.randn(4, world_size * h_per_rank, device=device, dtype=torch.float32)

packed = _pack(out, lse, world_size, h_per_rank, head_dim, None, None)
assert torch.isfinite(packed.float()).all()
67 changes: 62 additions & 5 deletions vllm/v1/attention/ops/dcp.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,13 +34,18 @@
# LSE/output combine


def mask_dcp_empty_shards_(
lse: torch.Tensor,
def _validate_dcp_empty_shard_args(
seq_lens: torch.Tensor | None,
query_start_loc: torch.Tensor | None,
) -> None:
) -> bool:
"""Validate the empty-shard mask inputs.

Returns True when masking is requested, False when it is disabled. Raises on
an inconsistent pair. Performs no device work so it is safe to call from a
CUDA-graph-captured region.
"""
if seq_lens is None and query_start_loc is None:
return
return False
if seq_lens is None or query_start_loc is None:
raise ValueError("seq_lens and query_start_loc must be provided together")
if (
Expand All @@ -49,6 +54,17 @@ def mask_dcp_empty_shards_(
or query_start_loc.shape[0] != seq_lens.shape[0] + 1
):
raise ValueError("query_start_loc must contain one boundary per sequence")
return True


def mask_dcp_empty_shards_(
lse: torch.Tensor,
seq_lens: torch.Tensor | None,
query_start_loc: torch.Tensor | None,
) -> None:
if not _validate_dcp_empty_shard_args(seq_lens, query_start_loc):
return
assert seq_lens is not None and query_start_loc is not None

row_indices = torch.arange(
lse.shape[0], device=lse.device, dtype=query_start_loc.dtype
Expand Down Expand Up @@ -442,6 +458,9 @@ def _dcp_a2a_pack_send_kernel(
out_ptr,
lse_ptr,
send_ptr,
seq_lens_ptr,
query_start_loc_ptr,
num_seqs,
out_stride_B,
out_stride_H,
out_stride_D,
Expand All @@ -455,11 +474,33 @@ def _dcp_a2a_pack_send_kernel(
HEAD_DIM: tl.constexpr,
H_PER_RANK: tl.constexpr,
LSE_PACK_DIM: tl.constexpr,
HAS_MASK: tl.constexpr,
NUM_SEQS_POW2: tl.constexpr,
):
batch_idx = tl.program_id(0).to(tl.int64)
local_head_idx = tl.program_id(1).to(tl.int64)
d_offsets = tl.arange(0, HEAD_DIM)

# Empty-shard masking, fused in to avoid a separate per-layer pass over the
# LSE. Rows that belong to a request with no local KV shard, and rows past
# the end of the batch (CUDA-graph padding), must not contribute to the
# combine, which the reduction achieves by giving them an LSE of -inf.
row_is_empty = False
if HAS_MASK:
# searchsorted(query_start_loc[1:], batch_idx, right=True), i.e. the
# number of request boundaries at or below this row.
seq_offsets = tl.arange(0, NUM_SEQS_POW2)
boundaries = tl.load(
query_start_loc_ptr + 1 + seq_offsets,
mask=seq_offsets < num_seqs,
other=2**31 - 1,
)
seq_idx = tl.sum((boundaries <= batch_idx).to(tl.int32), axis=0)
seq_idx = tl.minimum(seq_idx, num_seqs - 1)
num_rows = tl.load(query_start_loc_ptr + num_seqs)
seq_len = tl.load(seq_lens_ptr + seq_idx)
row_is_empty = (batch_idx >= num_rows) | (seq_len == 0)

for rank_idx in tl.static_range(N):
src_head_idx = rank_idx * H_PER_RANK + local_head_idx
send_base = (
Expand All @@ -481,6 +522,8 @@ def _dcp_a2a_pack_send_kernel(
lse_val = tl.load(
lse_ptr + batch_idx * lse_stride_B + src_head_idx * lse_stride_H
).to(tl.float32)
if HAS_MASK:
lse_val = tl.where(row_is_empty, -float("inf"), lse_val)
if LSE_PACK_DIM == 1:
tl.store(
send_ptr + send_base + HEAD_DIM * send_stride_D,
Expand Down Expand Up @@ -636,12 +679,21 @@ def _dcp_a2a_pack_send(
seq_lens: torch.Tensor | None = None,
query_start_loc: torch.Tensor | None = None,
) -> None:
mask_dcp_empty_shards_(cp_attn_lse, seq_lens, query_start_loc)
# The empty-shard mask is fused into the pack kernel below rather than
# applied as a separate pass: the mask is a pure function of seq_lens and
# query_start_loc, which are identical for every MLA layer in a step, while
# the eager form costs eight launch-bound kernels per layer.
has_mask = _validate_dcp_empty_shard_args(seq_lens, query_start_loc)
num_seqs = seq_lens.shape[0] if has_mask and seq_lens is not None else 0

grid = (cp_attn_out.shape[0], h_per_rank, 1)
_dcp_a2a_pack_send_kernel[grid](
cp_attn_out,
cp_attn_lse,
send_buffer,
seq_lens,
query_start_loc,
num_seqs,
cp_attn_out.stride(0),
cp_attn_out.stride(1),
cp_attn_out.stride(2),
Expand All @@ -655,6 +707,11 @@ def _dcp_a2a_pack_send(
HEAD_DIM=head_dim,
H_PER_RANK=h_per_rank,
LSE_PACK_DIM=lse_pack_dim,
HAS_MASK=has_mask,
# Only the arange bound is a constexpr: num_seqs itself is a runtime
# argument so the kernel specializes on at most log2(max_num_seqs)
# variants instead of one per distinct batch size.
NUM_SEQS_POW2=triton.next_power_of_2(max(num_seqs, 1)),
)


Expand Down
Loading