Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,47 @@ def test_scheduler_reports_allocation_failure(request_runner):
assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1


@pytest.mark.parametrize("async_scheduling", [True, False])
@pytest.mark.parametrize("prompt_offset", [-1, -2])
def test_last_block_offloaded_at_request_finish(
request_runner, async_scheduling: bool, prompt_offset: int
):
"""EOS fills the last block at request finish — verify req_status is kept alive.

prompt = block_size + prompt_offset tokens → not a full block at schedule time,
so _build_store_jobs creates no store job. After EOS, request_finished
keeps req_status alive so _build_store_jobs can process it on the next step.

prompt_offset=-1: EOS fills the block → store job created on next step.
prompt_offset=-2: block remains partial → no store job, cleanup in
_build_store_jobs deletes req_status.
"""
block_size = 4
runner = request_runner(
block_size=block_size,
num_gpu_blocks=10,
async_scheduling=async_scheduling,
)
# prompt = block_size + prompt_offset tokens
runner.new_request(token_ids=[0] * (block_size + prompt_offset))
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(list(keys))
)

# Run with one step (EOS)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
)

cs = runner.connector_scheduler
# Verify req_status is kept alive for _build_store_jobs to process
# regardless of whether there are storable blocks
assert "0" in cs._req_status, (
"req_status was deleted but should be kept alive "
"for _build_store_jobs to process finished_req_ids."
)


def test_scheduler_reports_lookup_sync_delay(request_runner):
runner = request_runner(
block_size=4,
Expand Down Expand Up @@ -560,11 +601,19 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo
# Group 1 (sliding window, window=2): only the last 2 blocks
# are within the window → loads blocks 1,2
expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)),
# The deferred store from the previous request's last block
# completes during this step, and its blocks are flushed because
# they were reallocated to the new request.
# Only block 1 (sliding window group) is stored — block 0's
# deferred store is flushed because it was reallocated.
expected_stored=((0, 1),),
expected_flushed=((0, 1),),
)

# one touch in get_num_new_matched_tokens x 2 groups
# 4 touch calls: 2 from get_num_new_matched_tokens (2 groups)
# + 2 from _get_reqs_to_store (2 groups)
touch_calls = runner.manager.touch.call_args_list
assert len(touch_calls) == 2
assert len(touch_calls) == 4
# full attention group touched all 3 blocks
assert len(touch_calls[0].args[0]) == 3
# sliding window group touched just the last 2 blocks
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import time
from collections.abc import Iterable, Sequence
from dataclasses import dataclass, field
from itertools import islice
from itertools import chain, islice
from typing import Any, NamedTuple

from vllm.config import VllmConfig
Expand Down Expand Up @@ -444,6 +444,24 @@ def _remove_pending_job(self, job_id: int, block_ids: list[int] | None) -> None:
if not pending:
del self._block_id_to_pending_jobs[bid]

def _calc_num_offloadable_tokens(
self, req_status: RequestOffloadState, num_computed_tokens: int
) -> int:
num = min(num_computed_tokens, req_status.req.num_tokens)
max_offload_tokens = req_status.max_offload_tokens
if max_offload_tokens is not None:
num = min(num, max_offload_tokens)
if self.config.offload_prompt_only:
num = min(num, req_status.req.num_prompt_tokens)
return num

def _maybe_cleanup_finished_req(
self, req_id: str, req_status: RequestOffloadState
) -> None:
"""Clean up req_status if finished and no in-flight jobs."""
if req_status.req.is_finished() and not req_status.transfer_jobs:
del self._req_status[req_id]

def _maximal_prefix_lookup(
self, keys: Iterable[OffloadKey], req_context: ReqContext
) -> int | None:
Expand Down Expand Up @@ -911,28 +929,24 @@ def _build_store_jobs(
) -> dict[int, TransferJob]:
blocks_per_chunk = self.config.blocks_per_chunk
store_jobs: dict[int, TransferJob] = {}
for req_id in scheduler_output.num_scheduled_tokens:
for req_id in chain(
scheduler_output.num_scheduled_tokens,
scheduler_output.finished_req_ids or (),
):
req_status = self._req_status.get(req_id)
if req_status is None:
continue
req = req_status.req

num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
# with async scheduling, some tokens may be missing
num_offloadable_tokens = min(num_tokens_after_batch, req.num_tokens)
max_offload_tokens = req_status.max_offload_tokens
if max_offload_tokens is not None:
num_offloadable_tokens = min(num_offloadable_tokens, max_offload_tokens)

# Skip decode-phase chunks: clamp to the prompt length so only
# prefill chunks become eligible for store. next_stored_chunk_idx
# never advances past this boundary, so decode chunks are never
# queued in this or any later step.
if self.config.offload_prompt_only:
num_offloadable_tokens = min(
num_offloadable_tokens, req.num_prompt_tokens
)
if req.is_finished():
num_tokens_after_batch = req.num_tokens
else:
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens

num_offloadable_tokens = self._calc_num_offloadable_tokens(
req_status, num_tokens_after_batch
)

# Filter out chunks skipped due to sliding window attention / SSM
# or unreachable by the load path's alignment constraints.
Expand Down Expand Up @@ -983,6 +997,7 @@ def _build_store_jobs(

if not new_offload_keys:
req_status.advance_stored_idx(num_offloadable_tokens)
self._maybe_cleanup_finished_req(req_id, req_status)
continue

store_output = self.manager.prepare_store(
Expand All @@ -993,10 +1008,12 @@ def _build_store_jobs(
_ConnectorMetricName.ALLOCATION_FAILURE
)
logger.warning("Request %s: cannot store chunks", req_id)
self._maybe_cleanup_finished_req(req_id, req_status)
continue

if not store_output.keys_to_store:
req_status.advance_stored_idx(num_offloadable_tokens)
self._maybe_cleanup_finished_req(req_id, req_status)
continue

self._touch(req_status)
Expand Down Expand Up @@ -1093,6 +1110,15 @@ def _build_store_jobs(
job_id,
)

if req.is_finished():
# Register non-sliding-window blocks for flush detection.
for bid in non_sliding_window_block_ids:
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
if bid in self._current_batch_allocated_block_ids:
self._current_batch_jobs_to_flush.add(job_id)

self._maybe_cleanup_finished_req(req_id, req_status)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This can be removed.
If we reached here, req_status.transfer_jobs is non-empty, and so self._maybe_cleanup_finished_req is a no-op.

return store_jobs

def build_connector_meta(
Expand Down Expand Up @@ -1255,8 +1281,6 @@ def request_finished(
Optional KVTransferParams to be included in the request outputs
returned by the engine.
"""
# TODO(orozery): possibly kickoff offload for last block
# which may have been deferred due to async scheduling
req_status = self._req_status.get(request.request_id)

if req_status is None:
Expand All @@ -1269,20 +1293,20 @@ def request_finished(

self.manager.on_request_finished(req_status.req_context)
self._maybe_observe_lookup_async_delay(req_status)
if not req_status.transfer_jobs:
# No in-flight jobs: no later complete_store()/complete_load() calls
# need this request's state.
del self._req_status[request.request_id]
return False, None

# In-flight jobs remain after the request stopped. Their completion may
# still call manager.complete_store()/complete_load(), so keep req_status.
# Pending stores outlive the request's block ownership; register them so
# future reuse of those blocks triggers a flush.
# Update offload keys with final block hash so _build_store_jobs can
# create store jobs for the last block(s) on the next schedule step.
req_status.update_offload_keys()

# Keep req_status alive: _build_store_jobs will process finished_req_ids
# on the next step and handle cleanup after creating store jobs.
# Register non_sliding_window_block_ids so future block reuse triggers
# a flush via _block_id_to_pending_jobs.
for job_id in req_status.transfer_jobs:
job_status = self._jobs[job_id]
for bid in job_status.non_sliding_window_block_ids or ():
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)

return False, None

def take_events(self) -> Iterable[KVCacheEvent]:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -280,7 +280,21 @@ def register_cross_layers_kv_cache(

def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata):
assert self.worker is not None

# Pop jobs_to_flush from store_jobs into _unsubmitted_store_jobs
# so the existing submission loop below submits them before wait().
if kv_connector_metadata.jobs_to_flush:
for job_id in kv_connector_metadata.jobs_to_flush:
entry = kv_connector_metadata.store_jobs.pop(job_id, None)
if entry is not None:
assert isinstance(entry.src_spec, GPULoadStoreSpec)
self._unsubmitted_store_jobs.append(
(job_id, entry.src_spec, entry.dst_spec)
)

# Submit deferred stores from previous step (and jobs_to_flush above).
for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs:
assert isinstance(src_spec, GPULoadStoreSpec)
success = self.worker.submit_store(job_id, src_spec, dst_spec)
assert success
self._unsubmitted_store_jobs.clear()
Expand Down
Loading