Skip to content
Closed
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
f806ad5
Fix unified HiCache physical transfers
Sep 2, 2026
329fb0c
Merge latest unified host-pool stack
Sep 2, 2026
f16293b
Support unified HiCache buffer staging
Sep 4, 2026
54d3892
Merge compacting unified host pool
Sep 4, 2026
136669c
Enable shared unified HiCache staging safely
Sep 4, 2026
63abe66
Merge unified decode host safety gates
Sep 4, 2026
f0af6be
Guard unified HiCache compatibility
Sep 4, 2026
1f8c4fe
Merge review fixes into HiCache transfers
Sep 4, 2026
d4462ed
Merge commit '372d2111cc' into yonghao/ump-hicache-physical-transfers
Sep 4, 2026
f417938
Merge commit 'ec0e7777f38d651b0c3e849d033b81985fc091b3' into yonghao/…
Sep 4, 2026
3c26277
Merge commit 'b0018def8d' into yonghao/ump-hicache-physical-transfers
Sep 4, 2026
18142d6
Keep physical reservation cleanup host-sync-free
Sep 4, 2026
95e9b6d
Merge host-pool updates and preserve unified transfer lifetimes
Sep 8, 2026
097e185
Merge UMP cleanup and restore host-sync-free physical release
Sep 8, 2026
b20d10f
Merge host pool refresh and adapt HiCache physical transfers
Sep 10, 2026
87df0bc
Merge Python 3.10 fixture compatibility fix from host pool
Sep 10, 2026
709e1df
Merge host-pool cleanup into physical transfers
Sep 10, 2026
3870148
Fix SWA-only backup and shared-host prefetch fallback
Sep 12, 2026
b03bcd2
Merge host-pool shared-capacity cleanup
Sep 13, 2026
dc84a3b
Merge shared host transfer cleanup
Sep 13, 2026
320d041
Separate storage hit allocation from queue policy
Sep 13, 2026
809821e
Keep short-prefix threshold policy at the cache boundary
Sep 13, 2026
242578b
Merge shared SWA budget cleanup
Sep 13, 2026
e951bb1
Merge host-pool fixes and preserve eviction progress under host pressure
Sep 13, 2026
1fcd125
Merge UMP cleanup and preserve non-unified HiCache failure semantics
Sep 13, 2026
a9a744a
Merge unified prefill rounding fix into HiCache transfers
Sep 14, 2026
e843a0a
Consolidate UMBP page layout and SWA host-pool dispatch
Sep 14, 2026
8e0df01
Merge host-pool cleanup preserving HiCache bindings
Sep 14, 2026
74b7f42
Merge updated unified host pools and preserve HiCache transfer fixes
Sep 14, 2026
acce638
Merge host-pool updates and simplify HiCache allocation policies
Sep 14, 2026
ccd3487
Fix Rust SWA backup sources and gate sidecar error recovery
Sep 14, 2026
d9c9565
Merge host-pool capability cleanup into HiCache transfers
Sep 14, 2026
a640ec3
Align HiCache prefix reuse with SWA tail allocation
Sep 14, 2026
bbbebd7
Merge shared KV budget fixes into HiCache physical transfers
Sep 14, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 42 additions & 10 deletions python/sglang/srt/arg_groups/kv_cache_hook.py
Original file line number Diff line number Diff line change
Expand Up @@ -261,10 +261,6 @@ def handle_unified_memory_pool(server_args: Any) -> None:
)
if cfg.disaggregation_decode_retraction_backup == "host_pool":
model_config = model_config_of(server_args)
assert not cfg.disaggregation_decode_enable_radix_cache, (
"--enable-unified-memory host-pool decode retraction does not "
"support decode radix-cache H2D/D2H transfers yet."
)
assert mambaish_config(model_config) is None, (
"--enable-unified-memory host-pool decode retraction does not "
"support hybrid-Mamba models."
Expand Down Expand Up @@ -304,13 +300,49 @@ def handle_unified_memory_pool(server_args: Any) -> None:
"write loc, so a captured decode replay raises. "
"TODO(ch-wan): carry out_cache_loc_virtual into the child view."
)
assert not (cfg.enable_hierarchical_cache or cfg.enable_lmcache), (
"--enable-unified-memory is not yet compatible with hierarchical / "
"host-tiered KV cache (--enable-hierarchical-cache / --enable-lmcache): "
"the unified-memory-pool init wires up no host pools, and its device mamba / "
"full-attention slots are VIRTUAL — the host-offload path does not "
"translate them to physical."
assert not cfg.enable_lmcache, (
"--enable-unified-memory is not yet compatible with --enable-lmcache."
)
assert not cfg.enable_unified_cache_external_linker, (
"--enable-unified-memory does not support "
"--enable-unified-cache-external-linker: direct L3 transfers do not "
"preserve unified page-envelope indices and compaction lifetimes. "
"Use --enable-hierarchical-cache for supported L2/L3 transfers."
)
if cfg.enable_hierarchical_cache:
model_config = model_config_of(server_args)
assert not use_mla_backend(server_args), (
"--enable-unified-memory with hierarchical cache does not support "
"MLA models yet."
)
assert mambaish_config(model_config) is None, (
"--enable-unified-memory with hierarchical cache does not support "
"recurrent-state models yet."
)
assert cfg.speculative_algorithm is None, (
"--enable-unified-memory with hierarchical cache does not support "
"speculative decoding yet."
)
assert cfg.hicache_io_backend in {"kernel", "direct"}, (
"--enable-unified-memory with hierarchical cache supports only "
"the kernel and direct I/O backends."
)
assert cfg.pp_size == 1, (
"--enable-unified-memory with hierarchical cache does not support "
"pipeline parallelism (--pp-size > 1)."
)
supported_storage_backends = {None, "file", "sim", "mori", "shm"}
if cfg.hicache_storage_backend not in supported_storage_backends:
raise ValueError(
"--enable-unified-memory with hierarchical cache does not "
"support this storage backend yet; supported backends are "
"file, sim, mori, and shm. Got "
f"{cfg.hicache_storage_backend!r}."
)
assert not envs.SGLANG_DISABLE_LAZY_COMPACTION.get(), (
"--enable-unified-memory with hierarchical cache requires lazy "
"compaction so pending H2D physical reservations remain stable."
)
if cfg.dcp_size > 1:
_validate_unified_memory_dcp(server_args)
# Only monolithic decode cuda-graph capture is wired; piecewise prefill
Expand Down
79 changes: 63 additions & 16 deletions python/sglang/srt/managers/cache_controller.py
Original file line number Diff line number Diff line change
Expand Up @@ -810,6 +810,9 @@ def start_writing(self) -> None:
completion = self.l2_transfer_engine.submit_device_to_host(
self._l2_transfers(host_indices, device_indices, pool_transfers)
)
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "write"), completion.finish_event
)

self.ack_write_queue.append(
HiCacheAck(
Expand Down Expand Up @@ -905,6 +908,11 @@ def _move_op_indices(
) -> tuple[torch.Tensor, torch.Tensor, Optional[List[PoolTransfer]]]:
return (*self.move_indices(op.host_indices, op.device_indices), None)

def _move_load_operation(
self, op: CacheOperation
) -> tuple[torch.Tensor, torch.Tensor, Optional[List[PoolTransfer]]]:
return self._move_op_indices(op)

def _l2_transfers(
self,
host_indices: torch.Tensor,
Expand Down Expand Up @@ -935,7 +943,7 @@ def start_loading(self) -> int:

producer_id = self.layer_done_counter.update_producer()
op = CacheOperation.merge_ops(self.load_queue)
host_indices, device_indices, pool_transfers = self._move_op_indices(op)
host_indices, device_indices, pool_transfers = self._move_load_operation(op)
self.load_queue.clear()
producer_event = self.layer_done_counter.events[producer_id]
producer_event.start_event.record()
Expand All @@ -954,6 +962,9 @@ def start_loading(self) -> int:
on_layer_done=producer_event.complete,
layer_num=self.layer_num,
)
self.mem_pool_device_allocator.set_hicache_transfer_done_event(
(id(self), "load"), completion.finish_event
)

self.ack_load_queue.append(
HiCacheAck(
Expand Down Expand Up @@ -1081,13 +1092,20 @@ def _page_transfer(self, operation: PrefetchOperation) -> int:
# Get one batch token, and update the completed_tokens if succeed
extra_info = HiCacheStorageExtraInfo(prefix_keys=prefix_keys)

hit_pages = self._page_transfer_kv_batch(
operation,
batch_hashes,
batch_host_indices,
extra_info,
kv_derived_transfers,
)
try:
hit_pages = self._page_transfer_kv_batch(
operation,
batch_hashes,
batch_host_indices,
extra_info,
kv_derived_transfers,
)
except Exception:
logger.exception(
"HiCache prefetch transfer failed for request %s",
Comment thread
ZYHowell marked this conversation as resolved.
Resolved
operation.request_id,
)
hit_pages = 0
# Check termination
if hit_pages != len(batch_hashes):
all_success = False
Expand Down Expand Up @@ -1148,19 +1166,20 @@ def prefetch_io_aux_func(self):
while not self.storage_stop_event.is_set():
try:
operation = self.prefetch_buffer.get(block=True, timeout=1)
if operation is None:
continue
except Empty:
continue
if operation is None:
continue
try:
self._page_transfer(operation)

finally:
self.prefetch_sync_queue.put(
PrefetchAck(
rid=operation.request_id,
completed_req=True,
operation=operation,
)
)
except Empty:
continue

def prefetch_rate_limited(self) -> bool:
"""
Expand All @@ -1182,6 +1201,24 @@ def prefetch_rate_limited(self) -> bool:
# todo: more sophisticated rate limiting based on storage backend performance
return False

def alloc_prefetch_host_buffers(
self, operation: StorageOperation, need_size: int
) -> Optional[torch.Tensor]:
"""Allocate the host bounce for a storage hit."""
return self.mem_pool_host.alloc(need_size)

def can_fit_prefetch_host_buffers(
self, operation: StorageOperation, need_size: int
) -> bool:
"""Whether a prefetch bounce can fit when its host pools are empty."""
return need_size <= self.mem_pool_host.size

def free_prefetch_host_buffers(
self, operation: StorageOperation, host_indices: torch.Tensor
) -> None:
"""Roll back a hit-sized host bounce before transfer ownership moves."""
self.mem_pool_host.free(host_indices)

def _storage_hit_query(self, operation) -> tuple[list[str], int]:
last_hash = operation.last_hash
tokens_to_fetch = operation.token_ids
Expand Down Expand Up @@ -1216,10 +1253,20 @@ def prefetch_thread_func(self):
operation = self.prefetch_queue.get(block=True, timeout=1)
if operation is None:
continue
if operation.is_terminated():
try:
if operation.is_terminated():
hash_value, storage_hit_count = [], 0
else:
hash_value, storage_hit_count = self._storage_hit_query(
operation
)
except Exception:
logger.exception(
"HiCache storage query failed for request %s",
operation.request_id,
)
hash_value, storage_hit_count = [], 0
else:
hash_value, storage_hit_count = self._storage_hit_query(operation)

storage_hit_count_tensor = torch.tensor(
storage_hit_count, dtype=torch.int
)
Expand Down
14 changes: 10 additions & 4 deletions python/sglang/srt/mem_cache/allocator/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from __future__ import annotations

import abc
from typing import TYPE_CHECKING, Protocol
from typing import TYPE_CHECKING, Hashable, Protocol

import torch

Expand Down Expand Up @@ -134,9 +134,11 @@ def merge_and_sort_free(self):
def translate_kv_indices_for_transfer(
self, kv_indices: torch.Tensor
) -> torch.Tensor:
"""Token ids as the PD transfer engine addresses them. Identity here
because a static pool's ids index its registered buffers directly;
virtual-id pools must override."""
"""Token ids as device transfer engines address them.

Identity here: a static pool's token ids index its registered buffers
directly. Virtual-id pools must override.
"""
return kv_indices

def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None):
Expand All @@ -147,6 +149,10 @@ def load_cpu_copy(
):
raise NotImplementedError()

def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
"""Record an asynchronous HiCache transfer completion event if needed."""
return

def alloc_extend(self, *args, **kwargs):
raise NotImplementedError("alloc_extend is only for paged allocator")

Expand Down
33 changes: 25 additions & 8 deletions python/sglang/srt/mem_cache/allocator/unified_hybrid_swa.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@

import logging
import math
from typing import Callable, List, Optional, Sequence, Tuple, TypeGuard
from typing import Callable, Hashable, List, Optional, Sequence, Tuple, TypeGuard

import torch
from torch.profiler import record_function
Expand Down Expand Up @@ -424,7 +424,8 @@ def can_reserve(

def _compaction_allowed(self) -> bool:
return all(
allocator.disagg_move_gate is None or allocator.disagg_move_gate()
allocator._pending_hicache_load_pages == 0
and (allocator.disagg_move_gate is None or allocator.disagg_move_gate())
for allocator in (self.full_attn_allocator, self.swa_attn_allocator)
)

Expand Down Expand Up @@ -561,7 +562,7 @@ def translate_kv_loc(
def translate_kv_indices_for_transfer(
self, kv_indices: torch.Tensor
) -> torch.Tensor:
"""Translate virtual token ids to full-pool physical ids for PD."""
"""Translate virtual token ids to raw full-pool physical token ids."""
return self.full_attn_allocator.translate_kv_loc(kv_indices.to(torch.int64))

def translate_swa_indices_for_transfer(
Expand Down Expand Up @@ -886,13 +887,25 @@ def free_full_segment(self, free_index: torch.Tensor, *, start_pos: int) -> None
def set_full_to_swa_mapping(
self, full_indices: torch.Tensor, swa_indices: torch.Tensor
) -> None:
"""No-op stub for HiCache load-back: in shared mode the swa v2p IS the
mapping, and HiCache for shared SWA is out of scope."""
return
if full_indices.numel() == 0:
return
assert full_indices.numel() == swa_indices.numel()
full_pages = full_indices.to(torch.int64) // self.page_size
swa_pages = swa_indices.to(torch.int64) // self.page_size
self.swa_attn_allocator.bind(full_pages, swa_pages)

def clear_full_to_swa_mapping(self, full_indices: torch.Tensor) -> None:
# Paired with set_full_to_swa_mapping: shared mode has no mapping tensor.
return
if full_indices.numel() == 0:
return
full_pages = torch.unique(full_indices.to(torch.int64) // self.page_size)
swa = self.swa_attn_allocator
physical_pages = swa.virtual_to_physical[full_pages].clone()
swa.virtual_to_physical.index_fill_(0, full_pages, -1)
live = physical_pages > 0
physical_pages = physical_pages[live]
full_pages = full_pages[live]
still_owned = swa.physical_to_virtual[physical_pages] == full_pages
swa.physical_to_virtual.index_fill_(0, physical_pages[still_owned], -1)

# -- free-group --

Expand Down Expand Up @@ -975,6 +988,10 @@ def clear(self) -> None:

# -- Lazy compaction hooks --

def set_hicache_transfer_done_event(self, transfer_key: Hashable, event) -> None:
self.full_attn_allocator.set_hicache_transfer_done_event(transfer_key, event)
self.swa_attn_allocator.set_hicache_transfer_done_event(transfer_key, event)

def set_latest_forward_done_event(self, event: Optional[torch.cuda.Event]) -> None:
"""Forward the per-batch `forward_done` event to BOTH sub-allocators."""
with record_function("UnifiedSWAAlloc.set_latest_forward_done_event"):
Expand Down
Loading
Loading