From 34786513805b36434dfe7620841b3048f33f9e98 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 17 Jul 2026 11:22:09 +0800 Subject: [PATCH 1/5] [Bugfix][KV Offloading] Offload last KV block at request finish When EOS fills the final KV cache block, that block was never offloaded because _build_store_jobs runs at schedule time (before the block is full) and request_finished had no store kickoff logic. Fix: process finished_req_ids in _build_store_jobs via itertools.chain, so the last block is offloaded on the next schedule step. request_finished calls update_offload_keys() and keeps req_status alive for deferred processing. Also fix handle_preemptions to submit jobs_to_flush before wait() to prevent block corruption race. Signed-off-by: Alex --- .../offloading_connector/test_scheduler.py | 53 +++- .../kv_connector/v1/offloading/scheduler.py | 283 ++++++++++-------- .../kv_connector/v1/offloading/worker.py | 14 + 3 files changed, 217 insertions(+), 133 deletions(-) diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index 017be3bbd5e1..bc5190db00f5 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -64,6 +64,47 @@ def test_scheduler_reports_allocation_failure(request_runner): assert reduced[_ConnectorMetricName.ALLOCATION_FAILURE] == 1 +@pytest.mark.parametrize("async_scheduling", [True, False]) +@pytest.mark.parametrize("prompt_offset", [-1, -2]) +def test_last_block_offloaded_at_request_finish( + request_runner, async_scheduling: bool, prompt_offset: int +): + """EOS fills the last block at request finish — verify req_status is kept alive. + + prompt = block_size + prompt_offset tokens → not a full block at schedule time, + so _build_store_jobs creates no store job. After EOS, request_finished + keeps req_status alive so _build_store_jobs can process it on the next step. + + prompt_offset=-1: EOS fills the block → store job created on next step. + prompt_offset=-2: block remains partial → no store job, cleanup in + _build_store_jobs deletes req_status. + """ + block_size = 4 + runner = request_runner( + block_size=block_size, + num_gpu_blocks=10, + async_scheduling=async_scheduling, + ) + # prompt = block_size + prompt_offset tokens + runner.new_request(token_ids=[0] * (block_size + prompt_offset)) + runner.manager.prepare_store.side_effect = lambda keys, req_context: ( + generate_store_output(list(keys)) + ) + + # Run with one step (EOS) + runner.run( + decoded_tokens=[EOS_TOKEN_ID], + ) + + cs = runner.connector_scheduler + # Verify req_status is kept alive for _build_store_jobs to process + # regardless of whether there are storable blocks + assert "0" in cs._req_status, ( + "req_status was deleted but should be kept alive " + "for _build_store_jobs to process finished_req_ids." + ) + + def test_scheduler_reports_lookup_sync_delay(request_runner): runner = request_runner( block_size=4, @@ -560,11 +601,19 @@ def test_two_groups_full_and_sliding_window(request_runner, async_scheduling: bo # Group 1 (sliding window, window=2): only the last 2 blocks # are within the window → loads blocks 1,2 expected_loaded=((0, 0), (0, 1), (0, 2), (1, 1), (1, 2)), + # The deferred store from the previous request's last block + # completes during this step, and its blocks are flushed because + # they were reallocated to the new request. + # Only block 1 (sliding window group) is stored — block 0's + # deferred store is flushed because it was reallocated. + expected_stored=((0, 1),), + expected_flushed=((0, 1),), ) - # one touch in get_num_new_matched_tokens x 2 groups + # 4 touch calls: 2 from get_num_new_matched_tokens (2 groups) + # + 2 from _get_reqs_to_store (2 groups) touch_calls = runner.manager.touch.call_args_list - assert len(touch_calls) == 2 + assert len(touch_calls) == 4 # full attention group touched all 3 blocks assert len(touch_calls[0].args[0]) == 3 # sliding window group touched just the last 2 blocks diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 3e0d09a10fd7..1302e0afa345 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -3,7 +3,7 @@ import time from collections.abc import Iterable, Sequence from dataclasses import dataclass, field -from itertools import islice +from itertools import chain, islice from typing import Any, NamedTuple from vllm.config import VllmConfig @@ -444,6 +444,17 @@ def _remove_pending_job(self, job_id: int, block_ids: list[int] | None) -> None: if not pending: del self._block_id_to_pending_jobs[bid] + def _calc_num_offloadable_tokens( + self, req_status: RequestOffloadState, num_computed_tokens: int + ) -> int: + num = min(num_computed_tokens, req_status.req.num_tokens) + max_offload_tokens = req_status.max_offload_tokens + if max_offload_tokens is not None: + num = min(num, max_offload_tokens) + if self.config.offload_prompt_only: + num = min(num, req_status.req.num_prompt_tokens) + return num + def _maximal_prefix_lookup( self, keys: Iterable[OffloadKey], req_context: ReqContext ) -> int | None: @@ -911,28 +922,25 @@ def _build_store_jobs( ) -> dict[int, TransferJob]: blocks_per_chunk = self.config.blocks_per_chunk store_jobs: dict[int, TransferJob] = {} - for req_id in scheduler_output.num_scheduled_tokens: + for req_id in chain( + scheduler_output.num_scheduled_tokens, + scheduler_output.finished_req_ids or (), + ): req_status = self._req_status.get(req_id) if req_status is None: continue req = req_status.req - num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] - num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens - # with async scheduling, some tokens may be missing - num_offloadable_tokens = min(num_tokens_after_batch, req.num_tokens) - max_offload_tokens = req_status.max_offload_tokens - if max_offload_tokens is not None: - num_offloadable_tokens = min(num_offloadable_tokens, max_offload_tokens) - - # Skip decode-phase chunks: clamp to the prompt length so only - # prefill chunks become eligible for store. next_stored_chunk_idx - # never advances past this boundary, so decode chunks are never - # queued in this or any later step. - if self.config.offload_prompt_only: - num_offloadable_tokens = min( - num_offloadable_tokens, req.num_prompt_tokens - ) + if req.is_finished(): + # Finished request: use final token count. + num_tokens_after_batch = req.num_tokens + else: + num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] + num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens + + num_offloadable_tokens = self._calc_num_offloadable_tokens( + req_status, num_tokens_after_batch + ) # Filter out chunks skipped due to sliding window attention / SSM # or unreachable by the load path's alignment constraints. @@ -981,117 +989,132 @@ def _build_store_jobs( continue new_offload_keys.append(offload_key) - if not new_offload_keys: - req_status.advance_stored_idx(num_offloadable_tokens) - continue - - store_output = self.manager.prepare_store( - new_offload_keys, req_status.req_context - ) - if store_output is None: - self._connector_stats.increase_counter( - _ConnectorMetricName.ALLOCATION_FAILURE - ) - logger.warning("Request %s: cannot store chunks", req_id) - continue - - if not store_output.keys_to_store: - req_status.advance_stored_idx(num_offloadable_tokens) - continue - - self._touch(req_status) - - keys_to_store = set(store_output.keys_to_store) - - group_sizes: list[int] = [] - block_indices: list[int] = [] - src_block_ids: list[int] = [] - sliding_window_block_ids: list[int] = [] - non_sliding_window_block_ids: list[int] = [] - for group_config, group_state in zip( - self.config.kv_group_configs, req_status.group_states - ): - is_sliding_window = ( - group_config.sliding_window_size_in_chunks is not None - ) - num_chunks = req_status.storable_chunks( - group_config, num_offloadable_tokens + should_advance = True + if new_offload_keys: + store_output = self.manager.prepare_store( + new_offload_keys, req_status.req_context ) - start_chunk_idx = group_state.next_stored_chunk_idx - block_ids = group_state.block_ids - num_group_blocks = 0 - start_gpu_block_idx: int | None = None - for idx, offload_key in enumerate( - group_state.offload_keys[start_chunk_idx:num_chunks] - ): - if offload_key not in keys_to_store: - continue + if store_output is None: + self._connector_stats.increase_counter( + _ConnectorMetricName.ALLOCATION_FAILURE + ) + logger.warning("Request %s: cannot store blocks", req_id) + # Don't advance_stored_idx: retry on next step + should_advance = False + elif store_output.keys_to_store: + self._touch(req_status) + + keys_to_store = set(store_output.keys_to_store) + + group_sizes: list[int] = [] + block_indices: list[int] = [] + src_block_ids: list[int] = [] + sliding_window_block_ids: list[int] = [] + non_sliding_window_block_ids: list[int] = [] + for group_config, group_state in zip( + self.config.kv_group_configs, req_status.group_states + ): + is_sliding_window = ( + group_config.sliding_window_size_in_chunks is not None + ) + num_chunks = req_status.storable_chunks( + group_config, num_offloadable_tokens + ) + start_chunk_idx = group_state.next_stored_chunk_idx + block_ids = group_state.block_ids + num_group_blocks = 0 + start_gpu_block_idx: int | None = None + for idx, offload_key in enumerate( + group_state.offload_keys[start_chunk_idx:num_chunks] + ): + if offload_key not in keys_to_store: + continue + + chunk_idx = start_chunk_idx + idx + + self._events_tracker.record_store( + req, group_config, chunk_idx, offload_key + ) + + gpu_block_idx = chunk_idx * blocks_per_chunk + for i in range(blocks_per_chunk): + block_id = block_ids[gpu_block_idx + i] + if block_id == 0: + continue + if start_gpu_block_idx is None: + start_gpu_block_idx = gpu_block_idx + i + src_block_ids.append(block_id) + num_group_blocks += 1 + if is_sliding_window: + sliding_window_block_ids.append(block_id) + else: + non_sliding_window_block_ids.append(block_id) + + group_sizes.append(num_group_blocks) + block_indices.append(start_gpu_block_idx or 0) + group_state.next_stored_chunk_idx = max( + group_state.next_stored_chunk_idx, num_chunks + ) - chunk_idx = start_chunk_idx + idx + src_spec = GPULoadStoreSpec( + src_block_ids, + group_sizes=group_sizes, + block_indices=block_indices, + ) + dst_spec = store_output.store_spec + + job_id = self._generate_job_id() + # a store can only be issued when no load is pending. + if req_status.transfer_jobs: + any_jid = next(iter(req_status.transfer_jobs)) + assert self._jobs[any_jid].is_store + req_status.transfer_jobs.add(job_id) + + # Watch sliding window blocks as they may get evicted + # before the request finishes + for bid in sliding_window_block_ids or (): + self._block_id_to_pending_jobs.setdefault(bid, set()).add( + job_id + ) - self._events_tracker.record_store( - req, group_config, chunk_idx, offload_key + # the non-sliding window blocks will be watched only + # when the request finishes + self._jobs[job_id] = TransferJobStatus( + req_id=req_id, + pending_count=self.config.num_workers, + keys=set(keys_to_store), + is_store=True, + non_sliding_window_block_ids=non_sliding_window_block_ids, + sliding_window_block_ids=sliding_window_block_ids or None, ) - gpu_block_idx = chunk_idx * blocks_per_chunk - for i in range(blocks_per_chunk): - block_id = block_ids[gpu_block_idx + i] - if block_id == 0: - continue - if start_gpu_block_idx is None: - start_gpu_block_idx = gpu_block_idx + i - src_block_ids.append(block_id) - num_group_blocks += 1 - if is_sliding_window: - sliding_window_block_ids.append(block_id) - else: - non_sliding_window_block_ids.append(block_id) - - group_sizes.append(num_group_blocks) - block_indices.append(start_gpu_block_idx or 0) - group_state.next_stored_chunk_idx = max( - group_state.next_stored_chunk_idx, num_chunks - ) + store_jobs[job_id] = TransferJob( + req_id=req_id, src_spec=src_spec, dst_spec=dst_spec + ) - src_spec = GPULoadStoreSpec( - src_block_ids, group_sizes=group_sizes, block_indices=block_indices - ) - dst_spec = store_output.store_spec - - job_id = self._generate_job_id() - # a store can only be issued when no load is pending. - if req_status.transfer_jobs: - any_jid = next(iter(req_status.transfer_jobs)) - assert self._jobs[any_jid].is_store - req_status.transfer_jobs.add(job_id) - - # Watch sliding window blocks as they may get evicted - # before the request finishes - for bid in sliding_window_block_ids or (): - self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) + logger.debug( + "Request %s offloading %s chunks upto %d tokens (job %d)", + req_id, + len(keys_to_store), + num_offloadable_tokens, + job_id, + ) - # the non-sliding window blocks will be watched only - # when the request finishes - self._jobs[job_id] = TransferJobStatus( - req_id=req_id, - pending_count=self.config.num_workers, - keys=set(keys_to_store), - is_store=True, - non_sliding_window_block_ids=non_sliding_window_block_ids, - sliding_window_block_ids=sliding_window_block_ids or None, - ) + if req.is_finished(): + # Register non-sliding-window blocks for flush detection. + for bid in non_sliding_window_block_ids: + self._block_id_to_pending_jobs.setdefault(bid, set()).add( + job_id + ) + if bid in self._current_batch_allocated_block_ids: + self._current_batch_jobs_to_flush.add(job_id) - store_jobs[job_id] = TransferJob( - req_id=req_id, src_spec=src_spec, dst_spec=dst_spec - ) + if should_advance: + req_status.advance_stored_idx(num_offloadable_tokens) - logger.debug( - "Request %s offloading %s chunks upto %d tokens (job %d)", - req_id, - len(keys_to_store), - num_offloadable_tokens, - job_id, - ) + # Unified cleanup + if req_status.req.is_finished() and not req_status.transfer_jobs: + del self._req_status[req_id] return store_jobs @@ -1255,8 +1278,6 @@ def request_finished( Optional KVTransferParams to be included in the request outputs returned by the engine. """ - # TODO(orozery): possibly kickoff offload for last block - # which may have been deferred due to async scheduling req_status = self._req_status.get(request.request_id) if req_status is None: @@ -1269,20 +1290,20 @@ def request_finished( self.manager.on_request_finished(req_status.req_context) self._maybe_observe_lookup_async_delay(req_status) - if not req_status.transfer_jobs: - # No in-flight jobs: no later complete_store()/complete_load() calls - # need this request's state. - del self._req_status[request.request_id] - return False, None - # In-flight jobs remain after the request stopped. Their completion may - # still call manager.complete_store()/complete_load(), so keep req_status. - # Pending stores outlive the request's block ownership; register them so - # future reuse of those blocks triggers a flush. + # Update offload keys with final block hash so _build_store_jobs can + # create store jobs for the last block(s) on the next schedule step. + req_status.update_offload_keys() + + # Keep req_status alive: _build_store_jobs will process finished_req_ids + # on the next step and handle cleanup after creating store jobs. + # Register non_sliding_window_block_ids so future block reuse triggers + # a flush via _block_id_to_pending_jobs. for job_id in req_status.transfer_jobs: job_status = self._jobs[job_id] for bid in job_status.non_sliding_window_block_ids or (): self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) + return False, None def take_events(self) -> Iterable[KVCacheEvent]: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py index e82e1e45c009..045e513c3f49 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/worker.py @@ -280,7 +280,21 @@ def register_cross_layers_kv_cache( def handle_preemptions(self, kv_connector_metadata: OffloadingConnectorMetadata): assert self.worker is not None + + # Pop jobs_to_flush from store_jobs into _unsubmitted_store_jobs + # so the existing submission loop below submits them before wait(). + if kv_connector_metadata.jobs_to_flush: + for job_id in kv_connector_metadata.jobs_to_flush: + entry = kv_connector_metadata.store_jobs.pop(job_id, None) + if entry is not None: + assert isinstance(entry.src_spec, GPULoadStoreSpec) + self._unsubmitted_store_jobs.append( + (job_id, entry.src_spec, entry.dst_spec) + ) + + # Submit deferred stores from previous step (and jobs_to_flush above). for job_id, src_spec, dst_spec in self._unsubmitted_store_jobs: + assert isinstance(src_spec, GPULoadStoreSpec) success = self.worker.submit_store(job_id, src_spec, dst_spec) assert success self._unsubmitted_store_jobs.clear() From 761b4a743c0239859bfb40db049c2efd5b8dda79 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 17 Jul 2026 12:10:40 +0800 Subject: [PATCH 2/5] [Refactor][KV Offloading] Use _maybe_cleanup_finished_req helper to minimize diff Restore flat continue structure in _build_store_jobs instead of should_advance flag + nested if/elif. Extract cleanup logic into _maybe_cleanup_finished_req helper called at each exit point. Reduces scheduler.py diff from 262 to 82 changed lines. Signed-off-by: Alex --- .../kv_connector/v1/offloading/scheduler.py | 241 +++++++++--------- 1 file changed, 124 insertions(+), 117 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 1302e0afa345..27a8eb32e22c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -455,6 +455,13 @@ def _calc_num_offloadable_tokens( num = min(num, req_status.req.num_prompt_tokens) return num + def _maybe_cleanup_finished_req( + self, req_id: str, req_status: RequestOffloadState + ) -> None: + """Clean up req_status if finished and no in-flight jobs.""" + if req_status.req.is_finished() and not req_status.transfer_jobs: + del self._req_status[req_id] + def _maximal_prefix_lookup( self, keys: Iterable[OffloadKey], req_context: ReqContext ) -> int | None: @@ -989,132 +996,132 @@ def _build_store_jobs( continue new_offload_keys.append(offload_key) - should_advance = True - if new_offload_keys: - store_output = self.manager.prepare_store( - new_offload_keys, req_status.req_context + if not new_offload_keys: + req_status.advance_stored_idx(num_offloadable_tokens) + self._maybe_cleanup_finished_req(req_id, req_status) + continue + + store_output = self.manager.prepare_store( + new_offload_keys, req_status.req_context + ) + if store_output is None: + self._connector_stats.increase_counter( + _ConnectorMetricName.ALLOCATION_FAILURE ) - if store_output is None: - self._connector_stats.increase_counter( - _ConnectorMetricName.ALLOCATION_FAILURE - ) - logger.warning("Request %s: cannot store blocks", req_id) - # Don't advance_stored_idx: retry on next step - should_advance = False - elif store_output.keys_to_store: - self._touch(req_status) - - keys_to_store = set(store_output.keys_to_store) - - group_sizes: list[int] = [] - block_indices: list[int] = [] - src_block_ids: list[int] = [] - sliding_window_block_ids: list[int] = [] - non_sliding_window_block_ids: list[int] = [] - for group_config, group_state in zip( - self.config.kv_group_configs, req_status.group_states - ): - is_sliding_window = ( - group_config.sliding_window_size_in_chunks is not None - ) - num_chunks = req_status.storable_chunks( - group_config, num_offloadable_tokens - ) - start_chunk_idx = group_state.next_stored_chunk_idx - block_ids = group_state.block_ids - num_group_blocks = 0 - start_gpu_block_idx: int | None = None - for idx, offload_key in enumerate( - group_state.offload_keys[start_chunk_idx:num_chunks] - ): - if offload_key not in keys_to_store: - continue - - chunk_idx = start_chunk_idx + idx - - self._events_tracker.record_store( - req, group_config, chunk_idx, offload_key - ) - - gpu_block_idx = chunk_idx * blocks_per_chunk - for i in range(blocks_per_chunk): - block_id = block_ids[gpu_block_idx + i] - if block_id == 0: - continue - if start_gpu_block_idx is None: - start_gpu_block_idx = gpu_block_idx + i - src_block_ids.append(block_id) - num_group_blocks += 1 - if is_sliding_window: - sliding_window_block_ids.append(block_id) - else: - non_sliding_window_block_ids.append(block_id) - - group_sizes.append(num_group_blocks) - block_indices.append(start_gpu_block_idx or 0) - group_state.next_stored_chunk_idx = max( - group_state.next_stored_chunk_idx, num_chunks - ) + logger.warning("Request %s: cannot store blocks", req_id) + # Don't advance_stored_idx: retry on next step + self._maybe_cleanup_finished_req(req_id, req_status) + continue - src_spec = GPULoadStoreSpec( - src_block_ids, - group_sizes=group_sizes, - block_indices=block_indices, - ) - dst_spec = store_output.store_spec - - job_id = self._generate_job_id() - # a store can only be issued when no load is pending. - if req_status.transfer_jobs: - any_jid = next(iter(req_status.transfer_jobs)) - assert self._jobs[any_jid].is_store - req_status.transfer_jobs.add(job_id) - - # Watch sliding window blocks as they may get evicted - # before the request finishes - for bid in sliding_window_block_ids or (): - self._block_id_to_pending_jobs.setdefault(bid, set()).add( - job_id - ) + if not store_output.keys_to_store: + req_status.advance_stored_idx(num_offloadable_tokens) + self._maybe_cleanup_finished_req(req_id, req_status) + continue - # the non-sliding window blocks will be watched only - # when the request finishes - self._jobs[job_id] = TransferJobStatus( - req_id=req_id, - pending_count=self.config.num_workers, - keys=set(keys_to_store), - is_store=True, - non_sliding_window_block_ids=non_sliding_window_block_ids, - sliding_window_block_ids=sliding_window_block_ids or None, - ) + self._touch(req_status) - store_jobs[job_id] = TransferJob( - req_id=req_id, src_spec=src_spec, dst_spec=dst_spec - ) + keys_to_store = set(store_output.keys_to_store) - logger.debug( - "Request %s offloading %s chunks upto %d tokens (job %d)", - req_id, - len(keys_to_store), - num_offloadable_tokens, - job_id, + group_sizes: list[int] = [] + block_indices: list[int] = [] + src_block_ids: list[int] = [] + sliding_window_block_ids: list[int] = [] + non_sliding_window_block_ids: list[int] = [] + for group_config, group_state in zip( + self.config.kv_group_configs, req_status.group_states + ): + is_sliding_window = ( + group_config.sliding_window_size_in_chunks is not None + ) + num_chunks = req_status.storable_chunks( + group_config, num_offloadable_tokens + ) + start_chunk_idx = group_state.next_stored_chunk_idx + block_ids = group_state.block_ids + num_group_blocks = 0 + start_gpu_block_idx: int | None = None + for idx, offload_key in enumerate( + group_state.offload_keys[start_chunk_idx:num_chunks] + ): + if offload_key not in keys_to_store: + continue + + chunk_idx = start_chunk_idx + idx + + self._events_tracker.record_store( + req, group_config, chunk_idx, offload_key ) - if req.is_finished(): - # Register non-sliding-window blocks for flush detection. - for bid in non_sliding_window_block_ids: - self._block_id_to_pending_jobs.setdefault(bid, set()).add( - job_id - ) - if bid in self._current_batch_allocated_block_ids: - self._current_batch_jobs_to_flush.add(job_id) + gpu_block_idx = chunk_idx * blocks_per_chunk + for i in range(blocks_per_chunk): + block_id = block_ids[gpu_block_idx + i] + if block_id == 0: + continue + if start_gpu_block_idx is None: + start_gpu_block_idx = gpu_block_idx + i + src_block_ids.append(block_id) + num_group_blocks += 1 + if is_sliding_window: + sliding_window_block_ids.append(block_id) + else: + non_sliding_window_block_ids.append(block_id) + + group_sizes.append(num_group_blocks) + block_indices.append(start_gpu_block_idx or 0) + group_state.next_stored_chunk_idx = max( + group_state.next_stored_chunk_idx, num_chunks + ) - if should_advance: - req_status.advance_stored_idx(num_offloadable_tokens) + src_spec = GPULoadStoreSpec( + src_block_ids, + group_sizes=group_sizes, + block_indices=block_indices, + ) + dst_spec = store_output.store_spec + + job_id = self._generate_job_id() + # a store can only be issued when no load is pending. + if req_status.transfer_jobs: + any_jid = next(iter(req_status.transfer_jobs)) + assert self._jobs[any_jid].is_store + req_status.transfer_jobs.add(job_id) + + # Watch sliding window blocks as they may get evicted + # before the request finishes + for bid in sliding_window_block_ids or (): + self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) - # Unified cleanup - if req_status.req.is_finished() and not req_status.transfer_jobs: - del self._req_status[req_id] + # the non-sliding window blocks will be watched only + # when the request finishes + self._jobs[job_id] = TransferJobStatus( + req_id=req_id, + pending_count=self.config.num_workers, + keys=set(keys_to_store), + is_store=True, + non_sliding_window_block_ids=non_sliding_window_block_ids, + sliding_window_block_ids=sliding_window_block_ids or None, + ) + + store_jobs[job_id] = TransferJob( + req_id=req_id, src_spec=src_spec, dst_spec=dst_spec + ) + + logger.debug( + "Request %s offloading %s chunks upto %d tokens (job %d)", + req_id, + len(keys_to_store), + num_offloadable_tokens, + job_id, + ) + + if req.is_finished(): + # Register non-sliding-window blocks for flush detection. + for bid in non_sliding_window_block_ids: + self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id) + if bid in self._current_batch_allocated_block_ids: + self._current_batch_jobs_to_flush.add(job_id) + + self._maybe_cleanup_finished_req(req_id, req_status) return store_jobs From 87755fd9d7b4eeafabd7a892e84a6a59fe50eb9c Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 17 Jul 2026 12:21:21 +0800 Subject: [PATCH 3/5] [Refactor][KV Offloading] Minimize scheduler.py diff MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert unrelated changes: logger message (blocks→chunks), GPULoadStoreSpec formatting, and inline comment. Keep only essential changes for last block offload fix. Signed-off-by: Alex --- .../kv_transfer/kv_connector/v1/offloading/scheduler.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 27a8eb32e22c..6813402362ea 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -939,7 +939,6 @@ def _build_store_jobs( req = req_status.req if req.is_finished(): - # Finished request: use final token count. num_tokens_after_batch = req.num_tokens else: num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id] @@ -1008,8 +1007,7 @@ def _build_store_jobs( self._connector_stats.increase_counter( _ConnectorMetricName.ALLOCATION_FAILURE ) - logger.warning("Request %s: cannot store blocks", req_id) - # Don't advance_stored_idx: retry on next step + logger.warning("Request %s: cannot store chunks", req_id) self._maybe_cleanup_finished_req(req_id, req_status) continue @@ -1073,9 +1071,7 @@ def _build_store_jobs( ) src_spec = GPULoadStoreSpec( - src_block_ids, - group_sizes=group_sizes, - block_indices=block_indices, + src_block_ids, group_sizes=group_sizes, block_indices=block_indices ) dst_spec = store_output.store_spec From 714fc7312cfe7c2a07cb820a19007f6cbb280e49 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 17 Jul 2026 12:37:08 +0800 Subject: [PATCH 4/5] fix Signed-off-by: Alex --- .../kv_transfer/kv_connector/v1/offloading/scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 6813402362ea..019fb501a404 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -1117,7 +1117,6 @@ def _build_store_jobs( if bid in self._current_batch_allocated_block_ids: self._current_batch_jobs_to_flush.add(job_id) - self._maybe_cleanup_finished_req(req_id, req_status) return store_jobs From 1e93aac6e83f64dab75282e88e17b1f2c9ab862b Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 17 Jul 2026 12:47:39 +0800 Subject: [PATCH 5/5] format Signed-off-by: Alex --- .../kv_transfer/kv_connector/v1/offloading/scheduler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 019fb501a404..5e98c1266e20 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -1117,7 +1117,6 @@ def _build_store_jobs( if bid in self._current_batch_allocated_block_ids: self._current_batch_jobs_to_flush.add(job_id) - return store_jobs def build_connector_meta(