Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions tests/v1/attention/test_attention_splitting.py
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,20 @@ def test_prefill_split_across_ubatches(
# Second ubatch: first request (continuation) seq_len should be full
# original
assert int(second_meta.seq_lens[0]) == seq_lens[split_req_idx]

# Tokens processed for the first chunk become computed context for the
# continuation. The original metadata must remain unchanged.
assert first_meta._num_computed_tokens_cpu is not None
assert second_meta._num_computed_tokens_cpu is not None
assert common._num_computed_tokens_cpu is not None
assert int(first_meta._num_computed_tokens_cpu[-1]) == context_lens[split_req_idx]
assert int(second_meta._num_computed_tokens_cpu[0]) == (
context_lens[split_req_idx] + tokens_in_first_chunk
)
assert int(common._num_computed_tokens_cpu[split_req_idx]) == context_lens[
split_req_idx
]

# Any following full requests in second ubatch should match originals
for j in range(1, second_meta.num_reqs):
# Map to original request index
Expand Down
5 changes: 5 additions & 0 deletions vllm/v1/worker/ubatch_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -296,6 +296,11 @@ def _make_metadata_with_slice(
if attn_metadata._num_computed_tokens_cpu is not None
else None
)
if splits_first_request and num_computed_tokens_cpu is not None:
# Tokens from the preceding microbatch are now part of this request's
# computed context. Clone to avoid mutating the parent metadata.
num_computed_tokens_cpu = num_computed_tokens_cpu.clone()
num_computed_tokens_cpu[0] += first_tok - start_locs[first_req]

if splits_last_request:
# NOTE: We use start_locs (the original query_start_loc_cpu) to calculate
Expand Down
Loading