From 2b35d1f3cb3dbfb869623afaa5fc1dc981b29c3a Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 15 Oct 2026 00:59:48 +0000 Subject: [PATCH 1/4] [Perf][Core] Skip the null head of sliding-window block tables A sliding-window block table keeps one null block per out-of-window block, so for long requests it is almost all null. Find that head by binary search and skip it when collecting unhashed block ids and when freeing a request. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Juntian Liu --- tests/v1/core/test_kv_cache_utils.py | 18 ++++++++++++++++++ vllm/v1/core/block_pool.py | 2 ++ vllm/v1/core/kv_cache_manager.py | 4 ++-- vllm/v1/core/kv_cache_utils.py | 17 +++++++++++++++++ vllm/v1/core/single_type_kv_cache_manager.py | 4 +++- 5 files changed, 42 insertions(+), 3 deletions(-) diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 1489315ec255..6a0b6393a225 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -52,6 +52,7 @@ hash_block_tokens, init_none_hash, make_block_hash_with_group_id, + null_head_len, tensor_data, to_event_extra_keys, ) @@ -2156,3 +2157,20 @@ def test_wrapped_mamba_group_requires_block_zeroing(): ) assert config.needs_kv_cache_zeroing + + +def test_null_head_len(): + null = KVCacheBlock(0, is_null=True) + real = [KVCacheBlock(i) for i in range(1, 4)] + tables = [ + [], + real, + [null] * 5, + [null] * 5 + real, + [null] * 2 + real[:1] + [null] * 3 + real[1:], # not one contiguous head + ] + for table in tables: + expected = next((i for i, b in enumerate(table) if not b.is_null), len(table)) + assert null_head_len(table) == expected + blocks = KVCacheBlocks(([null] * 4 + real, [null] + real[:1] + [null])) + assert blocks.get_unhashed_block_ids_all_groups() == [[1, 2, 3], [1]] diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index 3b0102fe84aa..e74e54b4b5d9 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -785,6 +785,8 @@ def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None: blocks_to_evict_first = [] other_pools: dict[BlockPool, list[KVCacheBlock]] = {} for block in ordered_blocks: + if block.is_null: # never pooled; its ref_cnt is not maintained + continue if block.pool is not None and block.pool is not self: other_pools.setdefault(block.pool, []).append(block) continue diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 4f472170a874..3602bf41195c 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -14,7 +14,7 @@ get_kv_cache_coordinator, ) from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector -from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy +from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy, null_head_len from vllm.v1.core.single_type_kv_cache_manager import MambaManager from vllm.v1.kv_cache_interface import ( AttentionSpec, @@ -117,7 +117,7 @@ def get_unhashed_block_ids_all_groups(self) -> list[list[int]]: return [ [ block.block_id - for block in group + for block in group[null_head_len(group) :] if block.block_hash is None and not block.is_null ] for group in self.blocks diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 5c51a8d131ff..1dccf8eed86c 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -216,6 +216,23 @@ def __repr__(self) -> str: ) +def null_head_len(blocks: Sequence[KVCacheBlock]) -> int: + """Number of leading null blocks, e.g. a sliding-window table's skipped prefix.""" + if not blocks or not blocks[0].is_null: + return 0 + lo, hi = 1, len(blocks) + while lo < hi: + mid = (lo + hi) // 2 + if blocks[mid].is_null: + lo = mid + 1 + else: + hi = mid + # All null entries are one shared object, so count() checks the run at C speed. + if blocks[:lo].count(blocks[0]) == lo: + return lo + return next(i for i, b in enumerate(blocks) if not b.is_null) + + class KVCacheBlockCopy(NamedTuple): src_block_id: int dst_block_id: int diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 28a7a1106f77..299bfec8f73c 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -15,6 +15,7 @@ BlockHashListWithBlockSize, BlockHashWithGroupId, KVCacheBlock, + null_head_len, resolve_block_hashes, ) from vllm.v1.hisparse.block_pool import SharedEventQueueBlockPool @@ -586,7 +587,8 @@ def free(self, request_id: str) -> None: """ # Free blocks in reverse order so that the tail blocks are freed first. - self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id))) + blocks = self.pop_blocks_for_free(request_id) + self.block_pool.free_blocks(reversed(blocks[null_head_len(blocks) :])) @classmethod @abstractmethod From 67eb4d72a56db0accd4754b739241fdf3580160d Mon Sep 17 00:00:00 2001 From: Juntian Liu Date: Thu, 15 Oct 2026 05:42:58 +0000 Subject: [PATCH 2/4] [Perf][Core] Trim the null head in pop_blocks_for_free Address review: move the null-head trim from free() into pop_blocks_for_free() (new trim_null_head(), no copy when there is no null head) so the deferred-free path benefits too, and trim the residency row HiSparseResidentManager returns. touch() skips the null block like free_blocks() does, which drops its now-redundant is_null check. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Juntian Liu --- .../core/test_single_type_kv_cache_manager.py | 32 +++++++++++++++++++ vllm/v1/core/block_pool.py | 6 ++-- vllm/v1/core/kv_cache_utils.py | 6 ++++ vllm/v1/core/single_type_kv_cache_manager.py | 12 +++---- 4 files changed, 48 insertions(+), 8 deletions(-) diff --git a/tests/v1/core/test_single_type_kv_cache_manager.py b/tests/v1/core/test_single_type_kv_cache_manager.py index cd4581dcefce..5f9f1abf4cd3 100644 --- a/tests/v1/core/test_single_type_kv_cache_manager.py +++ b/tests/v1/core/test_single_type_kv_cache_manager.py @@ -1012,3 +1012,35 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap(): f"but allocator pulled {len(new_blocks)}" ) total_computed = num_tokens + + +def test_pop_blocks_for_free_trims_null_head(): + block_size = 2 + spec = SlidingWindowSpec( + block_size=block_size, + num_kv_heads=1, + head_size=1, + dtype=torch.float32, + sliding_window=4, + ) + block_pool = BlockPool( + num_gpu_blocks=10, enable_caching=True, hash_block_size=block_size + ) + manager = get_sliding_window_manager(spec, block_pool) + null = block_pool.null_block + real = block_pool.get_new_blocks(2) + manager.req_to_blocks["swa"] = [null] * 3 + real + manager.req_to_blocks["dense"] = dense = block_pool.get_new_blocks(2) + + assert manager.pop_blocks_for_free("swa") == real + assert manager.pop_blocks_for_free("dense") is dense # no null head, no copy + assert "swa" not in manager.req_to_blocks + assert all(b.ref_cnt == 1 for b in real + dense) + + # The null block is never pooled, so touch / free_blocks leave it alone. + null_ref_cnt = null.ref_cnt + num_free_blocks = block_pool.get_num_free_blocks() + block_pool.touch([null]) + block_pool.free_blocks([null]) + assert null.ref_cnt == null_ref_cnt + assert block_pool.get_num_free_blocks() == num_free_blocks diff --git a/vllm/v1/core/block_pool.py b/vllm/v1/core/block_pool.py index e74e54b4b5d9..41839bd1f798 100644 --- a/vllm/v1/core/block_pool.py +++ b/vllm/v1/core/block_pool.py @@ -759,9 +759,11 @@ def touch(self, blocks: Sequence[KVCacheBlock]) -> None: """ for block in blocks: + if block.is_null: + continue # ref_cnt=0 means this block is in the free list (i.e. eviction # candidate), so remove it. - if block.ref_cnt == 0 and not block.is_null: + if block.ref_cnt == 0: self.free_block_queue.remove(block) block.ref_cnt += 1 if self.metrics_collector: @@ -791,7 +793,7 @@ def free_blocks(self, ordered_blocks: Iterable[KVCacheBlock]) -> None: other_pools.setdefault(block.pool, []).append(block) continue block.ref_cnt -= 1 - if block.ref_cnt == 0 and not block.is_null: + if block.ref_cnt == 0: if block.block_hash is None or not self.enable_caching: # LIFO reuse of non-cached blocks for better GPU locality. blocks_to_evict_first.append(block) diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 1dccf8eed86c..2ff5034d38cc 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -233,6 +233,12 @@ def null_head_len(blocks: Sequence[KVCacheBlock]) -> int: return next(i for i, b in enumerate(blocks) if not b.is_null) +def trim_null_head(blocks: list[KVCacheBlock]) -> list[KVCacheBlock]: + """``blocks`` without its leading null blocks; ``blocks`` itself if none.""" + nhl = null_head_len(blocks) + return blocks[nhl:] if nhl else blocks + + class KVCacheBlockCopy(NamedTuple): src_block_id: int dst_block_id: int diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index 299bfec8f73c..a583abb61e3e 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -15,8 +15,8 @@ BlockHashListWithBlockSize, BlockHashWithGroupId, KVCacheBlock, - null_head_len, resolve_block_hashes, + trim_null_head, ) from vllm.v1.hisparse.block_pool import SharedEventQueueBlockPool from vllm.v1.kv_cache_interface import ( @@ -570,14 +570,15 @@ def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: request_id: The request ID. Returns: - The request's blocks in allocation order. + The request's blocks in allocation order, without the leading + null blocks (e.g. a sliding-window table's skipped prefix). """ # Default to [] in case a request is freed (aborted) before alloc. req_blocks = self.req_to_blocks.pop(request_id, []) self.num_cached_block.pop(request_id, None) self._partial_hit_reqs.pop(request_id, None) - return req_blocks + return trim_null_head(req_blocks) def free(self, request_id: str) -> None: """Free the blocks for the request. @@ -587,8 +588,7 @@ def free(self, request_id: str) -> None: """ # Free blocks in reverse order so that the tail blocks are freed first. - blocks = self.pop_blocks_for_free(request_id) - self.block_pool.free_blocks(reversed(blocks[null_head_len(blocks) :])) + self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id))) @classmethod @abstractmethod @@ -2540,7 +2540,7 @@ def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: blocks = self.get_residency_row(request_id) self.residency_changes.pop(request_id, None) super().pop_blocks_for_free(request_id) - return blocks + return trim_null_head(blocks) def adopt_resident_page( self, request_id: str, block_idx: int, block: KVCacheBlock From 0f5df55aa0c5396ca08df5f1b3f4233ff54b9b59 Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 15 Oct 2026 11:53:07 -0700 Subject: [PATCH 3/4] Update vllm/v1/core/kv_cache_manager.py Signed-off-by: Nick Hill --- vllm/v1/core/kv_cache_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 3602bf41195c..11e0ff3ae006 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -117,7 +117,7 @@ def get_unhashed_block_ids_all_groups(self) -> list[list[int]]: return [ [ block.block_id - for block in group[null_head_len(group) :] + for block in trim_null_head(group) if block.block_hash is None and not block.is_null ] for group in self.blocks From e2f20de8f476208fceab399bde3739d869a8b2cd Mon Sep 17 00:00:00 2001 From: Nick Hill Date: Thu, 15 Oct 2026 14:02:52 -0700 Subject: [PATCH 4/4] fix import/typing --- vllm/v1/core/kv_cache_manager.py | 2 +- vllm/v1/core/kv_cache_utils.py | 15 +++++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 11e0ff3ae006..0ac1c388183d 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -14,7 +14,7 @@ get_kv_cache_coordinator, ) from vllm.v1.core.kv_cache_metrics import KVCacheMetricsCollector -from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy, null_head_len +from vllm.v1.core.kv_cache_utils import KVCacheBlock, KVCacheBlockCopy, trim_null_head from vllm.v1.core.single_type_kv_cache_manager import MambaManager from vllm.v1.kv_cache_interface import ( AttentionSpec, diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 2ff5034d38cc..aadc9cda5732 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -8,7 +8,15 @@ import os from collections.abc import Callable, Iterable, Iterator, Sequence from dataclasses import dataclass, field, replace -from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload +from typing import ( + TYPE_CHECKING, + Any, + NamedTuple, + NewType, + TypeAlias, + TypeVar, + overload, +) from vllm import envs from vllm.config import VllmConfig @@ -233,7 +241,10 @@ def null_head_len(blocks: Sequence[KVCacheBlock]) -> int: return next(i for i, b in enumerate(blocks) if not b.is_null) -def trim_null_head(blocks: list[KVCacheBlock]) -> list[KVCacheBlock]: +_BlockSeq = TypeVar("_BlockSeq", list[KVCacheBlock], Sequence[KVCacheBlock]) + + +def trim_null_head(blocks: _BlockSeq) -> _BlockSeq: """``blocks`` without its leading null blocks; ``blocks`` itself if none.""" nhl = null_head_len(blocks) return blocks[nhl:] if nhl else blocks