Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 2 additions & 7 deletions aiter/mla.py
Original file line number Diff line number Diff line change
Expand Up @@ -402,13 +402,8 @@ def _fold_seqlen_indptr(indptr, fold_factor):
"""Repeat each batch's seqlen ``fold_factor`` times (head-folding pseudo-batches)."""
lens = indptr[1:] - indptr[:-1]
folded_lens = lens.repeat_interleave(fold_factor)
out = torch.empty(
indptr.shape[0] + (fold_factor - 1) * (indptr.shape[0] - 1),
dtype=indptr.dtype,
device=indptr.device,
)
out[0] = 0
out[1:] = torch.cumsum(folded_lens, dim=0).to(indptr.dtype)
cumsum = torch.cumsum(folded_lens, dim=0).to(indptr.dtype)
out = torch.nn.functional.pad(cumsum, (1, 0), value=0)
return out


Expand Down
22 changes: 22 additions & 0 deletions op_tests/test_mla_persistent.py
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,24 @@ def check_support(dtype, kv_dtype, nhead):
return not (dtype == dtypes.bf16 and nhead == 32 and get_gfx() == "gfx942")


def check_fold_seqlen_indptr_cuda_graph_capture():
"""_fold_seqlen_indptr must be capturable into a CUDA graph (see PR desc)."""
indptr = torch.zeros(9, dtype=torch.int32, device="cuda")
indptr[1:] = torch.cumsum(
torch.randint(0, 4096, (8,), dtype=torch.int32, device="cuda"), dim=0
)

warmup_stream = torch.cuda.Stream()
warmup_stream.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(warmup_stream):
aiter.mla._fold_seqlen_indptr(indptr, 3)
torch.cuda.current_stream().wait_stream(warmup_stream)
torch.cuda.synchronize()

with torch.cuda.graph(torch.cuda.CUDAGraph()):
aiter.mla._fold_seqlen_indptr(indptr, 3)


def init_3buffer_kv_cache(
num_page: int,
page_size: int,
Expand Down Expand Up @@ -2080,3 +2098,7 @@ def test_absorb_decode_ds32_opus():
# df.to_csv(f"mla_nhead{nhead}decode_qlen{decode_qlen}.csv")
df_md = df.to_markdown(index=False)
aiter.logger.info("mla_persistent summary (markdown):\n%s", df_md)

if __name__ == "__main__":
check_fold_seqlen_indptr_cuda_graph_capture()
aiter.logger.info("_fold_seqlen_indptr cuda-graph capture: passed")
Loading