From 3cc811506b22ee91a77d960e7ec2b6e927a8902b Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Thu, 13 Aug 2026 16:06:28 -0400 Subject: [PATCH 1/4] [Core] Expose prefix cache retention interval Signed-off-by: Tyler Michael Smith --- tests/config/test_config_utils.py | 5 + tests/engine/test_arg_utils.py | 5 + tests/v1/core/test_prefix_caching.py | 104 ++++++++++-------- tests/v1/engine/test_engine_args.py | 5 + .../unit/test_mooncake_store_worker.py | 8 +- vllm/config/cache.py | 8 ++ .../kv_connector/v1/mooncake/store/worker.py | 2 +- vllm/engine/arg_utils.py | 8 ++ vllm/envs.py | 12 -- vllm/v1/core/kv_cache_coordinator.py | 22 +++- vllm/v1/core/kv_cache_manager.py | 4 +- vllm/v1/core/sched/scheduler.py | 1 + vllm/v1/simple_kv_offload/manager.py | 3 + 13 files changed, 123 insertions(+), 64 deletions(-) diff --git a/tests/config/test_config_utils.py b/tests/config/test_config_utils.py index 35bc1e167b52..24ef1b52a95d 100644 --- a/tests/config/test_config_utils.py +++ b/tests/config/test_config_utils.py @@ -216,6 +216,11 @@ def test_cache_config_hash_ignores_kv_cache_sizing_knobs(): assert CacheConfig(gpu_memory_utilization=0.5).compute_hash() == base_hash +def test_cache_config_hash_ignores_prefix_cache_retention_interval(): + base_hash = CacheConfig().compute_hash() + assert CacheConfig(prefix_cache_retention_interval=64).compute_hash() == base_hash + + def test_envs_compile_factors_relocation_invariant(tmp_path): """Relocating HOME or the XDG roots must not change the compile-cache env hash. diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index 2feb9f7a039d..b40f35c898ac 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -476,6 +476,7 @@ def test_prefix_cache_default(): # should be None by default (depends on model). engine_args = EngineArgs.from_cli_args(args=args) assert engine_args.enable_prefix_caching is None + assert engine_args.prefix_cache_retention_interval == 0 # with flag to turn it on. args = parser.parse_args(["--enable-prefix-caching"]) @@ -487,6 +488,10 @@ def test_prefix_cache_default(): engine_args = EngineArgs.from_cli_args(args=args) assert not engine_args.enable_prefix_caching + args = parser.parse_args(["--prefix-cache-retention-interval", "64"]) + engine_args = EngineArgs.from_cli_args(args=args) + assert engine_args.prefix_cache_retention_interval == 64 + @pytest.mark.parametrize( ("arg", "expected", "option"), diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 13ed7c7b9d8b..5bf9c5a40272 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -3119,9 +3119,8 @@ def test_hybrid_cache_blocks_clamped_to_lcm(): ) -def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch): +def test_hybrid_local_kv_retention_interval_aligns_in_manager(): """Verify fixed intervals retain sparse tails plus the latest replay tail.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64") block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3153,6 +3152,7 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch): max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=64, ) # The SWA manager uses the configured 64-token interval (a multiple of the @@ -3184,18 +3184,15 @@ def test_hybrid_local_kv_retention_interval_aligns_in_manager(monkeypatch): "interval, expected_match", [ # scheduler_block_size is 32 (= lcm(4*8, 8)); 33 is not a multiple of it. - ("33", "multiple of scheduler_block_size"), + (33, "multiple of scheduler_block_size"), # A negative multiple (-32 % 32 == 0) must still be rejected explicitly, # otherwise it would pass the modulo check and silently degrade to dense. - ("-32", "non-negative"), + (-32, "non-negative"), ], ) -def test_hybrid_local_kv_retention_interval_rejects_invalid( - monkeypatch, interval, expected_match -): +def test_hybrid_local_kv_retention_interval_rejects_invalid(interval, expected_match): """A retention interval that is negative or not a multiple of scheduler_block_size errors out at construction time.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", interval) block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3228,12 +3225,36 @@ def test_hybrid_local_kv_retention_interval_rejects_invalid( max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=interval, + ) + + +def test_zero_retention_is_ignored_for_full_attention(): + kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10) + manager = make_kv_cache_manager( + kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=16, + retention_interval=0, + ) + assert manager.coordinator.retention_interval == 0 + + +def test_positive_retention_rejects_full_attention(): + kv_cache_config = make_kv_cache_config(block_size=16, num_blocks=10) + with pytest.raises(ValueError, match="no sliding-window or Mamba"): + make_kv_cache_manager( + kv_cache_config, + max_model_len=8192, + enable_caching=True, + hash_block_size=16, + retention_interval=16, ) -def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch): +def test_hybrid_local_kv_retention_interval_survives_recycling(): """Verify retained local checkpoints are reused after block recycling.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "1024") hash_block_size = 4 kv_cache_config = KVCacheConfig( num_blocks=800, @@ -3286,6 +3307,7 @@ def test_hybrid_local_kv_retention_interval_survives_recycling(monkeypatch): max_model_len=4096, enable_caching=True, hash_block_size=hash_block_size, + retention_interval=1024, ) def fill_request(request_id: str, token_offset: int) -> list[int]: @@ -3314,9 +3336,8 @@ def fill_request(request_id: str, token_offset: int) -> list[int]: assert [len(blocks) for blocks in computed_blocks.blocks] == [4, 16, 128, 256] -def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatch): +def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(): """Verify latest-only retention reuses only the replayable prompt boundary.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3348,6 +3369,7 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=0, ) token_ids = [i for i in range(16) for _ in range(block_size)] @@ -3388,14 +3410,13 @@ def test_hybrid_local_kv_retention_latest_only_reuses_replay_boundary(monkeypatc assert len(computed_blocks.blocks[1]) == 0 -def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch): +def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(): """Verify MTP/EAGLE SWA retention keeps the extra proof block. EAGLE/MTP lookup matches one additional local block after the returned prefix and then drops it. Sparse retention must therefore cache the normal local tail at the latest replay boundary plus one extra SWA block. """ - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3428,6 +3449,7 @@ def test_hybrid_local_kv_retention_mtp_reuses_latest_boundary(monkeypatch): max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=0, use_eagle=True, ) @@ -3824,12 +3846,11 @@ def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate(): assert manager.get_blocks("test").get_block_ids() != ([], []) -def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch): - """Default path (no retention): freeing an SWA request must place its +def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(): + """Dense retention: freeing an SWA request must place its uncached scratch blocks at the front of the free queue (recycled first) and keep its cached checkpoint blocks at the back (retained for prefix hits). This split is always-on, independent of the retention interval.""" - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) block_size = 8 kv_cache_config = KVCacheConfig( num_blocks=100, @@ -3937,13 +3958,14 @@ def _make_pure_swa_manager(block_size, sliding_window, num_blocks=100, **kwargs) ) -def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch): +def test_pure_swa_retention_interval_caches_sparse_tails(): """Sparse retention must work for a pure-SWA single-group model, not just hybrid models: only the per-interval tails plus the latest replay tail are cached, and a replay still hits the latest replayable boundary.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64") block_size = 16 - manager = _make_pure_swa_manager(block_size, sliding_window=block_size) + manager = _make_pure_swa_manager( + block_size, sliding_window=block_size, retention_interval=64 + ) assert type(manager.coordinator).__name__ == "UnitaryKVCacheCoordinator" token_ids = [i for i in range(16) for _ in range(block_size)] @@ -3976,11 +3998,12 @@ def test_pure_swa_retention_interval_caches_sparse_tails(monkeypatch): assert num_computed == 240 -def test_pure_swa_retention_latest_only(monkeypatch): +def test_pure_swa_retention_latest_only(): """`=0` on a pure-SWA model keeps only the latest replay tail.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") block_size = 16 - manager = _make_pure_swa_manager(block_size, sliding_window=block_size) + manager = _make_pure_swa_manager( + block_size, sliding_window=block_size, retention_interval=0 + ) token_ids = [i for i in range(16) for _ in range(block_size)] req = make_request("0", token_ids, block_size, sha256) @@ -4008,10 +4031,9 @@ def test_pure_swa_retention_latest_only(monkeypatch): assert num_computed == 240 -def test_pure_swa_retention_dense_default_caches_all(monkeypatch): - """With retention unset, a pure-SWA model must keep the dense behavior: +def test_pure_swa_dense_retention_caches_all(): + """With retention set to ``None``, a pure-SWA model keeps dense behavior: every block boundary is a potential hit, so all blocks are cached.""" - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) block_size = 16 manager = _make_pure_swa_manager(block_size, sliding_window=block_size) @@ -4037,7 +4059,7 @@ def test_pure_swa_retention_dense_default_caches_all(monkeypatch): def test_mamba_reachable_block_mask_sparsifies_retention(): - """Mamba state-snapshot retention: with VLLM_PREFIX_CACHE_RETENTION_INTERVAL + """Mamba state-snapshot retention: with a configured retention interval, the manager keeps one cached state per interval-sized segment (plus the latest replay boundary) instead of a snapshot per block, which is what lets a small attention block_size avoid Mamba dominating the KV pool.""" @@ -4063,7 +4085,7 @@ def retained(retention_interval, num_prompt_tokens=256, end_block=16): ) return None if m is None else {i for i, v in enumerate(m) if v} - # Dense default (None) -> no mask, every block cached (unchanged behavior). + # Dense retention (None) -> no mask, every block cached. assert retained(None) is None # interval == block_size -> every block is a boundary -> stays dense. assert retained(block_size) is None @@ -4111,7 +4133,7 @@ def retained(retention_interval, shared_prefix_boundary, end_block=16): assert retained(0, 100) == {5, 14} # Coexists with segment tails (interval 64 -> {3,7,11,15} + replay 14). assert retained(64, 96) == {3, 5, 7, 11, 14, 15} - # Dense default ignores the hint (nothing to sparsify). + # Dense retention ignores the hint (nothing to sparsify). assert retained(None, 96) is None # Out-of-range boundary is a no-op (only replay 14 remains). assert retained(0, 16 * block_size * 2) == {14} @@ -4120,14 +4142,13 @@ def retained(retention_interval, shared_prefix_boundary, end_block=16): assert retained(0, None) == {14} -def test_mamba_shared_prefix_survives_zero_retention(monkeypatch): +def test_mamba_shared_prefix_survives_zero_retention(): """Manager-level check of the full wiring: a pinned shared-prefix boundary (``Request.shared_prefix_boundary``, set by the scheduler on Marconi-style detection) keeps its Mamba state block cached under - ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``, which otherwise retains only the + ``prefix_cache_retention_interval=0``, which otherwise retains only the end-of-prompt replay boundary. Without this, a shared prefix (junction before ``num_prompt``) would be recomputed by every sharing request.""" - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "0") block_size = 16 # 16-block (256-token) prompt; replay boundary is block 240 // 16 - 1 = 14. @@ -4140,6 +4161,7 @@ def cached_mamba_blocks(shared_prefix_boundary): max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=0, ) req = make_request("r", token_ids, block_size, sha256) req.shared_prefix_boundary = shared_prefix_boundary @@ -4163,24 +4185,21 @@ def cached_mamba_blocks(shared_prefix_boundary): assert cached_mamba_blocks(96) == {5, 14} -def test_mamba_shared_prefix_reuse_under_zero_retention(monkeypatch): +def test_mamba_shared_prefix_reuse_under_zero_retention(): """Full cross-request Marconi flow: a partial shared prefix cached by the detecting request must stay reusable by a later request under - ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without the pin the junction is + ``prefix_cache_retention_interval=0``. Without the pin the junction is masked out and the later request misses; with it (and under dense) the reuse is preserved.""" block_size = 16 def last_req_hit(retention, pin): - if retention is None: - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) - else: - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention)) manager = make_kv_cache_manager( _make_hybrid_kv_cache_config(block_size, 200, ["full", "mamba_align"]), max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=retention, ) shared = [7 for _ in range(2 * block_size)] # 2-block shared prefix @@ -4261,23 +4280,20 @@ def retained(retention, boundary, window, end_block=16): assert retained(0, 0, block_size) == {14} -def test_swa_shared_prefix_reuse_under_zero_retention(monkeypatch): +def test_swa_shared_prefix_reuse_under_zero_retention(): """SWA cross-request analog: a partial shared prefix's sliding-window tail - must stay reusable under ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL=0``. Without + must stay reusable under ``prefix_cache_retention_interval=0``. Without the pin the junction window is masked out and a later request misses; with it (and under dense) reuse is preserved.""" block_size = 16 def last_req_hit(retention, pin): - if retention is None: - monkeypatch.delenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", raising=False) - else: - monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", str(retention)) manager = make_kv_cache_manager( _make_hybrid_kv_cache_config(block_size, 200, ["full", "sliding_window"]), max_model_len=8192, enable_caching=True, hash_block_size=block_size, + retention_interval=retention, ) shared = [7 for _ in range(4 * block_size)] # 4-block shared prefix diff --git a/tests/v1/engine/test_engine_args.py b/tests/v1/engine/test_engine_args.py index 5033f4768bf7..b1ba1adcfd0a 100644 --- a/tests/v1/engine/test_engine_args.py +++ b/tests/v1/engine/test_engine_args.py @@ -18,6 +18,7 @@ def test_prefix_caching_from_cli(): assert vllm_config.cache_config.enable_prefix_caching, ( "V1 turns on prefix caching by default." ) + assert vllm_config.cache_config.prefix_cache_retention_interval == 0 # Turn it off possible with flag. args = parser.parse_args(["--no-enable-prefix-caching"]) @@ -47,6 +48,10 @@ def test_prefix_caching_from_cli(): with pytest.raises(ArgumentError): args = parser.parse_args(["--prefix-caching-hash-algo", "invalid"]) + args = parser.parse_args(["--prefix-cache-retention-interval", "64"]) + vllm_config = EngineArgs.from_cli_args(args=args).create_engine_config() + assert vllm_config.cache_config.prefix_cache_retention_interval == 64 + @pytest.mark.skipif(_xxhash is None, reason="xxhash not installed") def test_prefix_caching_xxhash_from_cli(): diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 034297100a7a..5c683df2c259 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -217,6 +217,7 @@ def _make_vllm_config( extra_config: dict[str, object] | None = None, rank: int = 0, decode_context_parallel_size: int = 1, + prefix_cache_retention_interval: int | None = 0, ) -> SimpleNamespace: return SimpleNamespace( model_config=_FakeModelConfig(), @@ -228,7 +229,11 @@ def _make_vllm_config( prefill_context_parallel_size=1, ), kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config), - cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10), + cache_config=SimpleNamespace( + block_size=16, + num_gpu_blocks=10, + prefix_cache_retention_interval=prefix_cache_retention_interval, + ), kv_events_config=SimpleNamespace(enable_kv_cache_events=False), speculative_config=None, ) @@ -1575,6 +1580,7 @@ def test_requester_worker_init_uses_positional_setup(tmp_path, monkeypatch): "mlx5_0", "10.0.0.7:50051", ) + assert w.coord.retention_interval == 0 def test_requester_worker_init_prefers_local_hostname_override( diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 654e9a53589b..3f419de69054 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -111,6 +111,13 @@ class CacheConfig: security risk tolerance against the performance benefits before turning this on. - "xxhash_cbor" combines canonical CBOR serialization with xxHash for reproducible hashing. Requires the optional ``xxhash`` package.""" + prefix_cache_retention_interval: int | None = Field(default=0, ge=0) + """Token interval between retained sliding-window and Mamba prefix-cache + checkpoints. ``0`` retains only semantic checkpoints, including the latest + replay boundary and shared-prefix junctions. Positive values additionally + retain periodic checkpoints at the specified interval, which must be a + multiple of the scheduler block size. ``None`` retains checkpoints densely. + Applies only to sliding-window and Mamba cache groups.""" kv_cache_dtype_skip_layers: list[str] = field(default_factory=list) """Layer patterns to skip KV cache quantization. Accepts layer indices (e.g., '0', '2', '4') or attention type names (e.g., 'sliding_window').""" @@ -217,6 +224,7 @@ def compute_hash(self) -> str: "num_gpu_blocks_override", "enable_prefix_caching", "prefix_caching_hash_algo", + "prefix_cache_retention_interval", # Prefix-caching implementation detail (doesn't affect compiled graph). "prefix_match_unit", "mamba_page_size_padded", diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 762e5eba9263..1c8ee81efa48 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1390,7 +1390,7 @@ def __init__( scheduler_block_size=self.block_size, hash_block_size=self.hash_block_size, use_eagle=use_eagle, - retention_interval=envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL, + retention_interval=self.cache_config.prefix_cache_retention_interval, ) # One ChunkedTokenDatabase per group; addresses populated in # register_kv_caches once the kv-cache layout is known. Each group's diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 528c884d73e3..b8777340a2c0 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -522,6 +522,9 @@ class EngineArgs: prefix_caching_hash_algo: PrefixCachingHashAlgo = ( CacheConfig.prefix_caching_hash_algo ) + prefix_cache_retention_interval: int | None = ( + CacheConfig.prefix_cache_retention_interval + ) disable_sliding_window: bool = ModelConfig.disable_sliding_window disable_cascade_attn: bool = ModelConfig.disable_cascade_attn offload_backend: str = OffloadConfig.offload_backend @@ -1228,6 +1231,10 @@ def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser: cache_group.add_argument( "--prefix-caching-hash-algo", **cache_kwargs["prefix_caching_hash_algo"] ) + cache_group.add_argument( + "--prefix-cache-retention-interval", + **cache_kwargs["prefix_cache_retention_interval"], + ) cache_group.add_argument( "--kv-cache-dtype-skip-layers", **cache_kwargs["kv_cache_dtype_skip_layers"] ) @@ -2006,6 +2013,7 @@ def create_engine_config( sliding_window=sliding_window, enable_prefix_caching=self.enable_prefix_caching, prefix_caching_hash_algo=self.prefix_caching_hash_algo, + prefix_cache_retention_interval=self.prefix_cache_retention_interval, kv_cache_dtype_skip_layers=self.kv_cache_dtype_skip_layers, kv_sharing_fast_prefill=self.kv_sharing_fast_prefill, mamba_cache_dtype=self.mamba_cache_dtype, diff --git a/vllm/envs.py b/vllm/envs.py index 5715508d12a2..c11e11417e0a 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -310,7 +310,6 @@ VLLM_LORA_ENABLE_DUAL_STREAM: bool = False VLLM_GPU_NIC_PCIE_MAPPING: str = "" VLLM_NIC_SELECTION_VARS: str = "" - VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None def get_default_cache_root(): @@ -1147,17 +1146,6 @@ def _resolve_rust_cli_path() -> str | None: if "VLLM_PLUGINS" not in os.environ else os.environ["VLLM_PLUGINS"].split(",") ), - # Retain local sliding-window KV checkpoints for prefix caching. - # Unset (default) preserves the dense local checkpointing behavior. `0` - # retains only the latest completed prompt boundary. Positive values retain - # checkpoints at the specified interval boundaries (rounded up to the - # prefix-cache alignment). - # Applies to sliding-window attention for now but not yet Mamba/linear attention. - "VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: ( - int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"]) - if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ - else None - ), # a local directory to look in for unrecognized LoRA adapters. # only works if plugins are enabled and # VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled. diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index 321bbb0a76ac..dd55f49b65b5 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -4,7 +4,6 @@ from collections.abc import Sequence from typing import NamedTuple -from vllm import envs from vllm.logger import init_logger from vllm.utils.math_utils import cdiv from vllm.v1.core.block_pool import BlockPool @@ -45,16 +44,18 @@ def _validate_prefix_cache_retention_interval( isinstance(g.kv_cache_spec, (SlidingWindowSpec, MambaSpec)) for g in kv_cache_config.kv_cache_groups ): + if retention_interval == 0: + return raise ValueError( - "VLLM_PREFIX_CACHE_RETENTION_INTERVAL is set but this model has " + "prefix_cache_retention_interval is set but this model has " "no sliding-window or Mamba KV cache group, so retention has no " - "effect. Unset it (it only applies to sliding-window and Mamba " + "effect. Set it to 0 (it only applies to sliding-window and Mamba " "attention)." ) if retention_interval < 0 or retention_interval % scheduler_block_size != 0: raise ValueError( - f"VLLM_PREFIX_CACHE_RETENTION_INTERVAL ({retention_interval}) " + f"prefix_cache_retention_interval ({retention_interval}) " "must be non-negative and a multiple of scheduler_block_size " f"({scheduler_block_size})." ) @@ -80,6 +81,7 @@ def __init__( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, + retention_interval: int | None = None, ): self.kv_cache_config = kv_cache_config self.max_model_len = max_model_len @@ -127,7 +129,7 @@ def __init__( # A positive retention interval must be a multiple of the base hit granularity # (``scheduler_block_size``) to land on real cache-hit boundaries. # 0 = keep only the latest replay boundary; None = dense; - self.retention_interval = envs.VLLM_PREFIX_CACHE_RETENTION_INTERVAL + self.retention_interval = retention_interval _validate_prefix_cache_retention_interval( self.retention_interval, self.scheduler_block_size, kv_cache_config ) @@ -407,6 +409,7 @@ def __init__( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, + retention_interval: int | None = None, ): super().__init__( kv_cache_config, @@ -419,6 +422,7 @@ def __init__( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, + retention_interval=retention_interval, metrics_collector=metrics_collector, ) self.num_single_type_manager = len(self.single_type_managers) @@ -457,6 +461,7 @@ def __init__( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, + retention_interval: int | None = None, ): super().__init__( kv_cache_config, @@ -469,6 +474,7 @@ def __init__( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, + retention_interval=retention_interval, metrics_collector=metrics_collector, ) self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec @@ -542,6 +548,7 @@ def __init__( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, + retention_interval: int | None = None, ): super().__init__( kv_cache_config, @@ -554,6 +561,7 @@ def __init__( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, + retention_interval=retention_interval, metrics_collector=metrics_collector, ) # hash_block_size: the block size used to compute block hashes. @@ -880,6 +888,7 @@ def get_kv_cache_coordinator( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, + retention_interval: int | None = None, ) -> KVCacheCoordinator: if not enable_caching: return KVCacheCoordinatorNoPrefixCache( @@ -892,6 +901,7 @@ def get_kv_cache_coordinator( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, + retention_interval=retention_interval, metrics_collector=metrics_collector, ) if len(kv_cache_config.kv_cache_groups) == 1: @@ -906,6 +916,7 @@ def get_kv_cache_coordinator( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, + retention_interval=retention_interval, metrics_collector=metrics_collector, ) return HybridKVCacheCoordinator( @@ -919,5 +930,6 @@ def get_kv_cache_coordinator( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, + retention_interval=retention_interval, metrics_collector=metrics_collector, ) diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index ca1fb73420a2..19ce3f56a22f 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -131,6 +131,7 @@ def __init__( pcp_world_size: int = 1, metrics_collector: KVCacheMetricsCollector | None = None, watermark: float = 0.0, + retention_interval: int | None = None, ) -> None: self.max_model_len = max_model_len # When unset, fall back to `max_model_len` so the recycling-aware cap @@ -160,6 +161,7 @@ def __init__( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, + retention_interval=retention_interval, metrics_collector=self.metrics_collector, ) self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) @@ -241,7 +243,7 @@ def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int, int - ``shared_prefix_boundary``: the block-aligned token position of a shared prefix that a sparse-retention group (Mamba / sliding window) has not cached yet (Marconi-style APC), or 0 if none. - Pinned so ``VLLM_PREFIX_CACHE_RETENTION_INTERVAL`` does not drop + Pinned so sparse prefix-cache retention does not drop the junction and defeat cross-request reuse. """ # We skip finding the prefix cache hit when prefix caching is diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 9d14d00edc56..06daf6672c5f 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -273,6 +273,7 @@ def __init__( pcp_world_size=1, scheduler_block_size=self.block_size, hash_block_size=hash_block_size, + retention_interval=self.cache_config.prefix_cache_retention_interval, metrics_collector=self.kv_metrics_collector, watermark=self.scheduler_config.watermark, ) diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 2e6839fef557..0e5cfeadb4eb 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -137,6 +137,9 @@ def __init__( pcp_world_size=1, scheduler_block_size=self.block_size, hash_block_size=self.hash_block_size, + retention_interval=( + vllm_config.cache_config.prefix_cache_retention_interval + ), ) self.cpu_block_pool: BlockPool = self.cpu_coordinator.block_pool # GPU block pool reference - bound after scheduler builds kv_cache_manager From 84882e97ec11893082b63d23ecc7e38f44086762 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Thu, 13 Aug 2026 16:19:55 -0400 Subject: [PATCH 2/4] [Core] Simplify prefix cache retention wiring Signed-off-by: Tyler Michael Smith --- tests/v1/core/test_contiguous_kv_packing.py | 2 ++ tests/v1/core/test_kv_cache_utils.py | 4 ++++ tests/v1/core/test_prefix_caching.py | 6 ++++++ .../kv_connector/unit/test_mooncake_store_worker.py | 12 +++++------- .../kv_connector/v1/mooncake/store/worker.py | 2 +- vllm/v1/core/kv_cache_coordinator.py | 13 +------------ vllm/v1/core/kv_cache_manager.py | 2 -- vllm/v1/core/kv_cache_utils.py | 6 ++++++ vllm/v1/core/sched/scheduler.py | 1 - vllm/v1/kv_cache_interface.py | 2 ++ vllm/v1/simple_kv_offload/manager.py | 10 +++------- 11 files changed, 30 insertions(+), 30 deletions(-) diff --git a/tests/v1/core/test_contiguous_kv_packing.py b/tests/v1/core/test_contiguous_kv_packing.py index 88d17e9acdc7..40906f621369 100644 --- a/tests/v1/core/test_contiguous_kv_packing.py +++ b/tests/v1/core/test_contiguous_kv_packing.py @@ -88,6 +88,7 @@ def _make_groups(n_c4, n_c128, n_swa): def _mock_vllm_config(kv_connector_extra_config: dict[str, str] | None = None): config = MagicMock() config.cache_config.num_gpu_blocks_override = None + config.cache_config.prefix_cache_retention_interval = 0 config.kv_transfer_config = None if kv_connector_extra_config is not None: config.kv_transfer_config = MagicMock() @@ -322,6 +323,7 @@ def test_hma_attention_groups_keep_default_backing(self): ) assert config.num_blocks == 32 + assert config.prefix_cache_retention_interval == 0 assert sum(t.size for t in config.kv_cache_tensors) == page_size * 2 * 32 assert config.kv_cache_tensors == [ KVCacheTensor(size=page_size * 32, shared_by=["full.0", "sw.0", "sw.1"]), diff --git a/tests/v1/core/test_kv_cache_utils.py b/tests/v1/core/test_kv_cache_utils.py index 33430ce85063..06d00d20df68 100644 --- a/tests/v1/core/test_kv_cache_utils.py +++ b/tests/v1/core/test_kv_cache_utils.py @@ -816,6 +816,7 @@ def test_metrics_empty_stats(): def test_get_kv_cache_configs_multiple_workers(): model_config = ModelConfig(max_model_len=16) vllm_config = VllmConfig(model_config=model_config) + vllm_config.cache_config.prefix_cache_retention_interval = None ref_kv_cache_spec = new_kv_cache_spec() same_kv_cache_specs = [ @@ -1173,6 +1174,7 @@ def test_get_kv_cache_configs_multiple_workers(): def test_get_kv_cache_configs_pp_sharding(asymmetric_memory): model_config = ModelConfig(max_model_len=512) vllm_config = VllmConfig(model_config=model_config) + vllm_config.cache_config.prefix_cache_retention_interval = None ref_kv_cache_spec = new_kv_cache_spec() pp_kv_cache_specs = [ @@ -1702,6 +1704,7 @@ def test_get_kv_cache_config_one_worker(): # pass max_model_len to pass check_enough_kv_cache_memory model_config = ModelConfig(max_model_len=16) vllm_config = VllmConfig(model_config=model_config) + vllm_config.cache_config.prefix_cache_retention_interval = None mem_per_block_per_layer = 16 * 2 * 64 * 4 * 2 # all layers are full attention -> single group @@ -2015,6 +2018,7 @@ def test_get_kv_cache_config_one_worker(): def test_get_kv_cache_configs_attention_free(): kv_cache_specs: dict[str, KVCacheSpec] = {} vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=16)) + vllm_config.cache_config.prefix_cache_retention_interval = None kv_cache_configs = get_kv_cache_configs(vllm_config, [kv_cache_specs], [0]) assert kv_cache_configs == [ KVCacheConfig( diff --git a/tests/v1/core/test_prefix_caching.py b/tests/v1/core/test_prefix_caching.py index 5bf9c5a40272..6cd117b1fa4a 100644 --- a/tests/v1/core/test_prefix_caching.py +++ b/tests/v1/core/test_prefix_caching.py @@ -4,6 +4,7 @@ import copy from collections.abc import Callable +from dataclasses import replace from math import lcm from types import SimpleNamespace @@ -110,6 +111,11 @@ def make_kv_cache_manager(kv_cache_config: KVCacheConfig, **kwargs) -> KVCacheMa "scheduler_block_size", lcm(*(g.kv_cache_spec.block_size for g in kv_cache_config.kv_cache_groups)), ) + if "retention_interval" in kwargs: + kv_cache_config = replace( + kv_cache_config, + prefix_cache_retention_interval=kwargs.pop("retention_interval"), + ) return KVCacheManager(kv_cache_config, **kwargs) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 5c683df2c259..b650ccaeebb3 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -217,7 +217,6 @@ def _make_vllm_config( extra_config: dict[str, object] | None = None, rank: int = 0, decode_context_parallel_size: int = 1, - prefix_cache_retention_interval: int | None = 0, ) -> SimpleNamespace: return SimpleNamespace( model_config=_FakeModelConfig(), @@ -229,17 +228,15 @@ def _make_vllm_config( prefill_context_parallel_size=1, ), kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config), - cache_config=SimpleNamespace( - block_size=16, - num_gpu_blocks=10, - prefix_cache_retention_interval=prefix_cache_retention_interval, - ), + cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10), kv_events_config=SimpleNamespace(enable_kv_cache_events=False), speculative_config=None, ) -def _make_kv_cache_config(*, block_size: int = 16) -> object: +def _make_kv_cache_config( + *, block_size: int = 16, prefix_cache_retention_interval: int | None = 0 +) -> object: """Minimal single-group KVCacheConfig for topology tests.""" from vllm.v1.kv_cache_interface import ( FullAttentionSpec, @@ -254,6 +251,7 @@ def _make_kv_cache_config(*, block_size: int = 16) -> object: num_blocks=10, kv_cache_tensors=[], kv_cache_groups=[KVCacheGroupSpec(["layer0"], spec)], + prefix_cache_retention_interval=prefix_cache_retention_interval, ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 1c8ee81efa48..64b80406d6d3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1390,7 +1390,7 @@ def __init__( scheduler_block_size=self.block_size, hash_block_size=self.hash_block_size, use_eagle=use_eagle, - retention_interval=self.cache_config.prefix_cache_retention_interval, + retention_interval=kv_cache_config.prefix_cache_retention_interval, ) # One ChunkedTokenDatabase per group; addresses populated in # register_kv_caches once the kv-cache layout is known. Each group's diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index dd55f49b65b5..4d4020c27e39 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -81,7 +81,6 @@ def __init__( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, - retention_interval: int | None = None, ): self.kv_cache_config = kv_cache_config self.max_model_len = max_model_len @@ -129,7 +128,7 @@ def __init__( # A positive retention interval must be a multiple of the base hit granularity # (``scheduler_block_size``) to land on real cache-hit boundaries. # 0 = keep only the latest replay boundary; None = dense; - self.retention_interval = retention_interval + self.retention_interval = kv_cache_config.prefix_cache_retention_interval _validate_prefix_cache_retention_interval( self.retention_interval, self.scheduler_block_size, kv_cache_config ) @@ -409,7 +408,6 @@ def __init__( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, - retention_interval: int | None = None, ): super().__init__( kv_cache_config, @@ -422,7 +420,6 @@ def __init__( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, - retention_interval=retention_interval, metrics_collector=metrics_collector, ) self.num_single_type_manager = len(self.single_type_managers) @@ -461,7 +458,6 @@ def __init__( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, - retention_interval: int | None = None, ): super().__init__( kv_cache_config, @@ -474,7 +470,6 @@ def __init__( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, - retention_interval=retention_interval, metrics_collector=metrics_collector, ) self.kv_cache_spec = self.kv_cache_config.kv_cache_groups[0].kv_cache_spec @@ -548,7 +543,6 @@ def __init__( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, - retention_interval: int | None = None, ): super().__init__( kv_cache_config, @@ -561,7 +555,6 @@ def __init__( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, - retention_interval=retention_interval, metrics_collector=metrics_collector, ) # hash_block_size: the block size used to compute block hashes. @@ -888,7 +881,6 @@ def get_kv_cache_coordinator( scheduler_block_size: int, hash_block_size: int, metrics_collector: KVCacheMetricsCollector | None = None, - retention_interval: int | None = None, ) -> KVCacheCoordinator: if not enable_caching: return KVCacheCoordinatorNoPrefixCache( @@ -901,7 +893,6 @@ def get_kv_cache_coordinator( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, - retention_interval=retention_interval, metrics_collector=metrics_collector, ) if len(kv_cache_config.kv_cache_groups) == 1: @@ -916,7 +907,6 @@ def get_kv_cache_coordinator( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, - retention_interval=retention_interval, metrics_collector=metrics_collector, ) return HybridKVCacheCoordinator( @@ -930,6 +920,5 @@ def get_kv_cache_coordinator( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, - retention_interval=retention_interval, metrics_collector=metrics_collector, ) diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 19ce3f56a22f..50c3f36483f4 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -131,7 +131,6 @@ def __init__( pcp_world_size: int = 1, metrics_collector: KVCacheMetricsCollector | None = None, watermark: float = 0.0, - retention_interval: int | None = None, ) -> None: self.max_model_len = max_model_len # When unset, fall back to `max_model_len` so the recycling-aware cap @@ -161,7 +160,6 @@ def __init__( pcp_world_size=pcp_world_size, scheduler_block_size=scheduler_block_size, hash_block_size=hash_block_size, - retention_interval=retention_interval, metrics_collector=self.metrics_collector, ) self.num_kv_cache_groups = len(kv_cache_config.kv_cache_groups) diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 57d6600b368e..d4baaf2322b8 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -1347,6 +1347,9 @@ def get_kv_cache_config_from_groups( num_blocks=1, kv_cache_tensors=[], kv_cache_groups=kv_cache_groups, + prefix_cache_retention_interval=( + vllm_config.cache_config.prefix_cache_retention_interval + ), ) # Determine how model runners should initialize the KV cache tensors. @@ -1406,6 +1409,9 @@ def get_kv_cache_config_from_groups( num_blocks=num_blocks, kv_cache_tensors=kv_cache_tensors, kv_cache_groups=kv_cache_groups, + prefix_cache_retention_interval=( + vllm_config.cache_config.prefix_cache_retention_interval + ), ) diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 06daf6672c5f..9d14d00edc56 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -273,7 +273,6 @@ def __init__( pcp_world_size=1, scheduler_block_size=self.block_size, hash_block_size=hash_block_size, - retention_interval=self.cache_config.prefix_cache_retention_interval, metrics_collector=self.kv_metrics_collector, watermark=self.scheduler_config.watermark, ) diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index dcc8298781b5..9e0aa27dc7d3 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -1024,6 +1024,8 @@ class KVCacheConfig: For models with multiple types of attention, there will be multiple groups, see `_get_kv_cache_config_uniform_page_size` for more details. """ + prefix_cache_retention_interval: int | None = None + """Resolved retention policy for local prefix-cache checkpoints.""" @property def has_mamba_layers(self) -> bool: diff --git a/vllm/v1/simple_kv_offload/manager.py b/vllm/v1/simple_kv_offload/manager.py index 0e5cfeadb4eb..1e4fcae68cf9 100644 --- a/vllm/v1/simple_kv_offload/manager.py +++ b/vllm/v1/simple_kv_offload/manager.py @@ -4,7 +4,7 @@ import contextlib from collections.abc import Iterable -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from typing import TYPE_CHECKING, Any from vllm.config import VllmConfig @@ -137,9 +137,6 @@ def __init__( pcp_world_size=1, scheduler_block_size=self.block_size, hash_block_size=self.hash_block_size, - retention_interval=( - vllm_config.cache_config.prefix_cache_retention_interval - ), ) self.cpu_block_pool: BlockPool = self.cpu_coordinator.block_pool # GPU block pool reference - bound after scheduler builds kv_cache_manager @@ -193,7 +190,6 @@ def _derive_cpu_config( """Derive a CPU KVCacheConfig from the GPU config. Same kv_cache_groups, num_blocks scaled by CPU/GPU memory ratio.""" # Import here to avoid potential circular imports - from vllm.v1.kv_cache_interface import KVCacheConfig as KVCacheConfigCls from vllm.v1.kv_cache_interface import KVCacheTensor assert len(gpu_config.kv_cache_tensors) > 0 @@ -218,10 +214,10 @@ def _derive_cpu_config( for t in gpu_config.kv_cache_tensors ] - return KVCacheConfigCls( + return replace( + gpu_config, num_blocks=num_cpu_blocks, kv_cache_tensors=cpu_tensors, - kv_cache_groups=gpu_config.kv_cache_groups, ) @staticmethod From e9b1bf707034a760ff1c8592922419da432c4067 Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Thu, 13 Aug 2026 16:45:57 -0400 Subject: [PATCH 3/4] [Core] Preserve prefix cache retention env compatibility Co-authored-by: OpenAI Codex Signed-off-by: Tyler Michael Smith --- tests/engine/test_arg_utils.py | 18 ++++++++++++++++++ vllm/config/cache.py | 17 +++++++++++++++-- vllm/engine/arg_utils.py | 4 ++-- vllm/envs.py | 6 ++++++ 4 files changed, 41 insertions(+), 4 deletions(-) diff --git a/tests/engine/test_arg_utils.py b/tests/engine/test_arg_utils.py index b40f35c898ac..6a799ee912d4 100644 --- a/tests/engine/test_arg_utils.py +++ b/tests/engine/test_arg_utils.py @@ -493,6 +493,24 @@ def test_prefix_cache_default(): assert engine_args.prefix_cache_retention_interval == 64 +def test_prefix_cache_retention_interval_from_deprecated_env( + monkeypatch, caplog, disable_log_dedup +): + monkeypatch.setenv("VLLM_PREFIX_CACHE_RETENTION_INTERVAL", "64") + + engine_args = EngineArgs() + + assert engine_args.prefix_cache_retention_interval == 64 + assert "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in caplog.text + assert "deprecated" in caplog.text + assert "prefix_cache_retention_interval" in caplog.text + + parser = EngineArgs.add_cli_args(FlexibleArgumentParser()) + args = parser.parse_args(["--prefix-cache-retention-interval", "32"]) + engine_args = EngineArgs.from_cli_args(args) + assert engine_args.prefix_cache_retention_interval == 32 + + @pytest.mark.parametrize( ("arg", "expected", "option"), [ diff --git a/vllm/config/cache.py b/vllm/config/cache.py index 3f419de69054..430303e7aba9 100644 --- a/vllm/config/cache.py +++ b/vllm/config/cache.py @@ -7,7 +7,7 @@ from pydantic import Field, field_validator, model_validator -from vllm.config.utils import config +from vllm.config.utils import config, get_from_deprecated_env_if_set from vllm.logger import init_logger from vllm.utils.torch_utils import ( is_quantized_kv_cache, @@ -35,6 +35,17 @@ "nvfp4", "nvfp4_4over6", ] + + +def _get_prefix_cache_retention_interval() -> int | None: + env_value = get_from_deprecated_env_if_set( + "VLLM_PREFIX_CACHE_RETENTION_INTERVAL", + "v0.29", + "prefix_cache_retention_interval", + ) + return 0 if env_value is None else int(env_value) + + MambaDType = Literal["auto", "float32", "float16", "bfloat16"] MambaCacheMode = Literal["all", "align", "none"] PrefixCachingHashAlgo = Literal["sha256", "sha256_cbor", "xxhash", "xxhash_cbor"] @@ -111,7 +122,9 @@ class CacheConfig: security risk tolerance against the performance benefits before turning this on. - "xxhash_cbor" combines canonical CBOR serialization with xxHash for reproducible hashing. Requires the optional ``xxhash`` package.""" - prefix_cache_retention_interval: int | None = Field(default=0, ge=0) + prefix_cache_retention_interval: int | None = Field( + default_factory=_get_prefix_cache_retention_interval, ge=0 + ) """Token interval between retained sliding-window and Mamba prefix-cache checkpoints. ``0`` retains only semantic checkpoints, including the latest replay boundary and shared-prefix junctions. Positive values additionally diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index b8777340a2c0..15920393a91a 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -522,8 +522,8 @@ class EngineArgs: prefix_caching_hash_algo: PrefixCachingHashAlgo = ( CacheConfig.prefix_caching_hash_algo ) - prefix_cache_retention_interval: int | None = ( - CacheConfig.prefix_cache_retention_interval + prefix_cache_retention_interval: int | None = get_field( + CacheConfig, "prefix_cache_retention_interval" ) disable_sliding_window: bool = ModelConfig.disable_sliding_window disable_cascade_attn: bool = ModelConfig.disable_cascade_attn diff --git a/vllm/envs.py b/vllm/envs.py index c11e11417e0a..033cdc7f32ea 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -310,6 +310,7 @@ VLLM_LORA_ENABLE_DUAL_STREAM: bool = False VLLM_GPU_NIC_PCIE_MAPPING: str = "" VLLM_NIC_SELECTION_VARS: str = "" + VLLM_PREFIX_CACHE_RETENTION_INTERVAL: int | None = None def get_default_cache_root(): @@ -1146,6 +1147,11 @@ def _resolve_rust_cli_path() -> str | None: if "VLLM_PLUGINS" not in os.environ else os.environ["VLLM_PLUGINS"].split(",") ), + "VLLM_PREFIX_CACHE_RETENTION_INTERVAL": lambda: ( + int(os.environ["VLLM_PREFIX_CACHE_RETENTION_INTERVAL"]) + if "VLLM_PREFIX_CACHE_RETENTION_INTERVAL" in os.environ + else None + ), # a local directory to look in for unrecognized LoRA adapters. # only works if plugins are enabled and # VLLM_ALLOW_RUNTIME_LORA_UPDATING is enabled. From 53280a71ecafa6d618bc5c7d3419194bdc22fcdf Mon Sep 17 00:00:00 2001 From: Tyler Michael Smith Date: Fri, 14 Aug 2026 13:05:54 -0400 Subject: [PATCH 4/4] Fix hybrid KV event test retention Co-authored-by: OpenAI Codex Signed-off-by: Tyler Michael Smith --- tests/v1/engine/test_engine_core_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/v1/engine/test_engine_core_client.py b/tests/v1/engine/test_engine_core_client.py index 64adf7a8b3c8..9b9d3f7b4a5d 100644 --- a/tests/v1/engine/test_engine_core_client.py +++ b/tests/v1/engine/test_engine_core_client.py @@ -1066,6 +1066,7 @@ def test_kv_cache_events( model=model_name, enforce_eager=True, enable_prefix_caching=True, + prefix_cache_retention_interval=None, block_size=block_size, ) engine_args.kv_events_config = publisher_config