Skip to content

[Bugfix][KV Offload] Reset the store cursor when a request restarts from token 0 #63010

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
orozery merged 2 commits into from
Oct 25, 2026
Merged

[Bugfix][KV Offload] Reset the store cursor when a request restarts from token 0 #63010

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
5afea9b
[Bugfix][KV Offload] Reset the store cursor when a request restarts f…
AdvayMonga Oct 20, 2026
fd865ee
Merge branch 'main' into fix/offload-reset-store-cursor-on-preemption
vllm-agent Oct 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 17 additions & 8 deletions tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1846,7 +1846,9 @@ def test_offloading_connector(request_runner, async_scheduling: bool):


@pytest.mark.parametrize("async_scheduling", [True, False])
def test_request_preemption(request_runner, async_scheduling: bool):
@pytest.mark.parametrize("cpu_hit", [True, False])
def test_request_preemption(request_runner, async_scheduling: bool, cpu_hit: bool):
"""Resume reloads the chunks, or re-stores them if the CPU tier lost them."""
block_size = 4
blocks_per_chunk = 3
tokens_per_chunk = block_size * blocks_per_chunk
Expand Down Expand Up @@ -1897,16 +1899,23 @@ def test_request_preemption(request_runner, async_scheduling: bool):
free_block_queue.num_free_blocks = num_free_blocks_empty
runner.scheduler.reset_prefix_cache()

# request should now return from preemption
# re-load [0, ..., 8] from the CPU and store [9, 10, 11]
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 3
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[0] * block_size,
expected_loaded=(0, 1, 2, 3, 4, 5, 6, 7, 8),
)
if cpu_hit:
# request should now return from preemption
# re-load [0, ..., 8] from the CPU and store [9, 10, 11]
runner.connector_scheduler._maximal_prefix_lookup = lambda keys, ctx, *_: 3
runner.run(
decoded_tokens=[0] * block_size,
expected_loaded=(0, 1, 2, 3, 4, 5, 6, 7, 8),
)
else:
# lookup misses: recompute and re-store [0, ..., 8], then store [9, 10, 11]
runner.run(
decoded_tokens=[0] * block_size,
expected_stored=(0, 1, 2, 3, 4, 5, 6, 7, 8),
)

runner.run(
decoded_tokens=[EOS_TOKEN_ID],
Expand Down
2 changes: 2 additions & 0 deletions vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1246,6 +1246,8 @@ def get_num_new_matched_tokens(
req_status = self._req_status[request.request_id]
for group_state in req_status.group_states:
group_state.block_ids.clear()
# Restarting at token 0: offer every chunk again.
group_state.next_stored_chunk_idx = 0

if req_status.transfer_jobs:
logger.debug(
Expand Down
Loading