diff --git a/python/sglang/srt/disaggregation/decode.py b/python/sglang/srt/disaggregation/decode.py index 9ac2121e3759..715536c76403 100644 --- a/python/sglang/srt/disaggregation/decode.py +++ b/python/sglang/srt/disaggregation/decode.py @@ -90,9 +90,11 @@ from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.mem_cache.memory_pool import ( HybridReqToTokenPool, - KVCache, ReqToTokenPool, ) +from sglang.srt.mem_cache.pool.base import ( + KVCache, +) from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.observability.req_time_stats import ( set_schedule_time_batch, diff --git a/python/sglang/srt/disaggregation/prefill.py b/python/sglang/srt/disaggregation/prefill.py index b91ea8161ce6..ca64eb0a0de8 100644 --- a/python/sglang/srt/disaggregation/prefill.py +++ b/python/sglang/srt/disaggregation/prefill.py @@ -90,7 +90,7 @@ from torch.distributed import ProcessGroup from sglang.srt.managers.scheduler import GenerationBatchResult, Scheduler - from sglang.srt.mem_cache.memory_pool import KVCache + from sglang.srt.mem_cache.pool.base import KVCache logger = logging.getLogger(__name__) diff --git a/python/sglang/srt/disaggregation/utils.py b/python/sglang/srt/disaggregation/utils.py index 9c4acca59ce2..d7cb14138065 100644 --- a/python/sglang/srt/disaggregation/utils.py +++ b/python/sglang/srt/disaggregation/utils.py @@ -1294,7 +1294,6 @@ def setup_state_kv_args( """ from sglang.srt.disaggregation.base.conn import StateType from sglang.srt.hardware_backend.npu.memory_pool_npu import NPUMLATokenToKVPool - from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.mem_cache.memory_pool import ( DSATokenToKVPool, @@ -1302,6 +1301,7 @@ def setup_state_kv_args( MHATokenToKVPoolMXFP8, MiniMaxSparseKVPool, ) + from sglang.srt.mem_cache.pool.base import BaseSWAKVPool from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool kv_args.state_types = [] diff --git a/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py b/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py index d69e0511b12a..4cadb6ab4704 100644 --- a/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py +++ b/python/sglang/srt/hardware_backend/mlx/model_runner_stub.py @@ -14,7 +14,8 @@ MlxAuxiliaryStateReqToTokenPool, ) from sglang.srt.mem_cache.allocator import TokenToKVPoolAllocator -from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.pool.base import KVCache from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.model_executor.model_runner_components.layer_setup import ( ModelLayerInfo, diff --git a/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py b/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py index fc855da78cb1..986a6b48eff6 100644 --- a/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py +++ b/python/sglang/srt/hardware_backend/musa/attention/flashattention_backend.py @@ -22,7 +22,7 @@ from sglang.srt.layers.utils.cp_utils import ( cp_allgather_and_save_kv_cache, ) -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.runtime_context import get_schedule if TYPE_CHECKING: diff --git a/python/sglang/srt/hardware_backend/npu/allocator_npu.py b/python/sglang/srt/hardware_backend/npu/allocator_npu.py index f425b85ce71f..36974698c336 100644 --- a/python/sglang/srt/hardware_backend/npu/allocator_npu.py +++ b/python/sglang/srt/hardware_backend/npu/allocator_npu.py @@ -9,7 +9,7 @@ from sglang.srt.utils import get_num_new_pages, next_power_of_2 if TYPE_CHECKING: - from sglang.srt.mem_cache.memory_pool import KVCache + from sglang.srt.mem_cache.pool.base import KVCache class NPUPagedTokenToKVPoolAllocator(PagedTokenToKVPoolAllocator): diff --git a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py index 8f131c352bfd..43af268058a3 100644 --- a/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py +++ b/python/sglang/srt/hardware_backend/npu/attention/ascend_backend.py @@ -27,7 +27,7 @@ from sglang.srt.layers.attention.dsa.utils import is_dsa_enable_prefill_cp from sglang.srt.layers.radix_attention import AttentionType from sglang.srt.layers.utils.cp_utils import cp_all_gather_rerange_kv_cache -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import ( diff --git a/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py b/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py index a562c7ccd5dc..6fc04c691281 100644 --- a/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py +++ b/python/sglang/srt/hardware_backend/npu/memory_pool_npu.py @@ -10,6 +10,8 @@ MiniMaxSparseKVPool, MLATokenToKVPool, get_tensor_size_bytes, +) +from sglang.srt.mem_cache.pool.base import ( unwrap_write_loc, ) from sglang.srt.utils import get_bool_env_var diff --git a/python/sglang/srt/kv_canary/pool_patcher/api.py b/python/sglang/srt/kv_canary/pool_patcher/api.py index b73599ff807d..4c088f1a2ac9 100644 --- a/python/sglang/srt/kv_canary/pool_patcher/api.py +++ b/python/sglang/srt/kv_canary/pool_patcher/api.py @@ -13,10 +13,12 @@ from sglang.srt.kv_canary.pool_patcher.buffer_alloc import resolve_real_kv_read_bytes from sglang.srt.mem_cache.deepseek_v4_memory_pool import DeepSeekV4TokenToKVPool from sglang.srt.mem_cache.memory_pool import ( - KVCache, MHATokenToKVPool, MHATokenToKVPoolFP4, ) +from sglang.srt.mem_cache.pool.base import ( + KVCache, +) from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool logger = logging.getLogger(__name__) diff --git a/python/sglang/srt/layers/attention/aiter_backend.py b/python/sglang/srt/layers/attention/aiter_backend.py index 93188aceb0a4..8b6890582cdc 100755 --- a/python/sglang/srt/layers/attention/aiter_backend.py +++ b/python/sglang/srt/layers/attention/aiter_backend.py @@ -92,7 +92,7 @@ forward_decode_vectorized_5d, forward_extend_vectorized_5d, ) -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.utils import get_bool_env_var diff --git a/python/sglang/srt/layers/attention/flashattention_backend.py b/python/sglang/srt/layers/attention/flashattention_backend.py index 79d100be922e..9e9e8ac24a25 100644 --- a/python/sglang/srt/layers/attention/flashattention_backend.py +++ b/python/sglang/srt/layers/attention/flashattention_backend.py @@ -26,7 +26,7 @@ from sglang.srt.layers.cp.base import CPAttentionBackendKind, get_cp_strategy from sglang.srt.layers.cp.utils import is_cp_active from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import ( diff --git a/python/sglang/srt/layers/attention/flashinfer_backend.py b/python/sglang/srt/layers/attention/flashinfer_backend.py index 140a571c7685..b487feb1dd67 100644 --- a/python/sglang/srt/layers/attention/flashinfer_backend.py +++ b/python/sglang/srt/layers/attention/flashinfer_backend.py @@ -34,8 +34,7 @@ KVCacheAttentionAccessKind, ) from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import BaseSWAKVPool, KVWriteLoc from sglang.srt.model_executor.cuda_graph_config import ( Backend, Phase, diff --git a/python/sglang/srt/layers/attention/hpc_ops_backend.py b/python/sglang/srt/layers/attention/hpc_ops_backend.py index f5fb08c09d42..b9c1f55870cc 100644 --- a/python/sglang/srt/layers/attention/hpc_ops_backend.py +++ b/python/sglang/srt/layers/attention/hpc_ops_backend.py @@ -42,7 +42,7 @@ ) from sglang.srt.compilation.compilation_config import register_split_op from sglang.srt.layers.attention.base_attn_backend import AttentionBackend -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.model_executor.forward_context import get_attn_backend from sglang.srt.model_executor.runner_backend_utils.breakable_cuda_graph import ( diff --git a/python/sglang/srt/layers/attention/intel_amx_backend.py b/python/sglang/srt/layers/attention/intel_amx_backend.py index c2c79b7977f2..fd52c272840b 100644 --- a/python/sglang/srt/layers/attention/intel_amx_backend.py +++ b/python/sglang/srt/layers/attention/intel_amx_backend.py @@ -6,7 +6,7 @@ from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.runtime_context import get_parallel, get_spec diff --git a/python/sglang/srt/layers/attention/torch_native_backend.py b/python/sglang/srt/layers/attention/torch_native_backend.py index 89e1d8518791..61fcd60416d7 100644 --- a/python/sglang/srt/layers/attention/torch_native_backend.py +++ b/python/sglang/srt/layers/attention/torch_native_backend.py @@ -7,7 +7,7 @@ from sglang.srt.layers.attention.base_attn_backend import AttentionBackend from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch diff --git a/python/sglang/srt/layers/attention/triton_backend.py b/python/sglang/srt/layers/attention/triton_backend.py index 8046af61b0a3..aa9ef5a1343f 100644 --- a/python/sglang/srt/layers/attention/triton_backend.py +++ b/python/sglang/srt/layers/attention/triton_backend.py @@ -29,8 +29,7 @@ get_dcp_lens, ) from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import BaseSWAKVPool, KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.cuda_graph_config import ( Backend, diff --git a/python/sglang/srt/layers/attention/trtllm_mha_backend.py b/python/sglang/srt/layers/attention/trtllm_mha_backend.py index 59291270f806..109d9d18ac0d 100644 --- a/python/sglang/srt/layers/attention/trtllm_mha_backend.py +++ b/python/sglang/srt/layers/attention/trtllm_mha_backend.py @@ -42,7 +42,7 @@ KVCacheAttentionAccessKind, ) from sglang.srt.layers.radix_attention import AttentionType -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import ( diff --git a/python/sglang/srt/layers/attention/xpu_backend.py b/python/sglang/srt/layers/attention/xpu_backend.py index 5573bc477c33..abdb732aa3c0 100644 --- a/python/sglang/srt/layers/attention/xpu_backend.py +++ b/python/sglang/srt/layers/attention/xpu_backend.py @@ -12,7 +12,7 @@ merge_state_v2_wrapper, prepare_swa_spec_page_table_triton, ) -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardBatch, ForwardMode from sglang.srt.runtime_context import ( diff --git a/python/sglang/srt/layers/cp/zigzag.py b/python/sglang/srt/layers/cp/zigzag.py index 868c527c667e..ac4ebc51f0c4 100644 --- a/python/sglang/srt/layers/cp/zigzag.py +++ b/python/sglang/srt/layers/cp/zigzag.py @@ -51,7 +51,7 @@ from sglang.srt.layers.dp_attention import ( is_allocation_symmetric, ) -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.model_executor.forward_context import get_token_to_kv_pool from sglang.srt.runtime_context import get_device, get_parallel diff --git a/python/sglang/srt/mem_cache/allocator/base.py b/python/sglang/srt/mem_cache/allocator/base.py index fa640b650672..77b9b2388ad3 100644 --- a/python/sglang/srt/mem_cache/allocator/base.py +++ b/python/sglang/srt/mem_cache/allocator/base.py @@ -21,7 +21,7 @@ import torch if TYPE_CHECKING: - from sglang.srt.mem_cache.memory_pool import KVCache + from sglang.srt.mem_cache.pool.base import KVCache class MambaFullCacheDonor(Protocol): diff --git a/python/sglang/srt/mem_cache/allocator/paged.py b/python/sglang/srt/mem_cache/allocator/paged.py index ffc479e8ce24..e26e3a03428d 100755 --- a/python/sglang/srt/mem_cache/allocator/paged.py +++ b/python/sglang/srt/mem_cache/allocator/paged.py @@ -34,7 +34,7 @@ _is_hip = is_hip() if TYPE_CHECKING: - from sglang.srt.mem_cache.memory_pool import KVCache + from sglang.srt.mem_cache.pool.base import KVCache def alloc_extend_naive( diff --git a/python/sglang/srt/mem_cache/allocator/swa.py b/python/sglang/srt/mem_cache/allocator/swa.py index 676f2450a004..795bcc2c1157 100644 --- a/python/sglang/srt/mem_cache/allocator/swa.py +++ b/python/sglang/srt/mem_cache/allocator/swa.py @@ -5,7 +5,7 @@ from sglang.srt.mem_cache.allocator.base import BaseTokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.paged import PagedTokenToKVPoolAllocator from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator -from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool +from sglang.srt.mem_cache.pool.base import BaseSWAKVPool from sglang.srt.utils import is_npu from sglang.srt.utils.common import get_num_new_pages from sglang.srt.utils.invariants import Bucket, Invariant, IsTrue, expect diff --git a/python/sglang/srt/mem_cache/allocator/token.py b/python/sglang/srt/mem_cache/allocator/token.py index 4782a1de2856..782f2d66ae1e 100644 --- a/python/sglang/srt/mem_cache/allocator/token.py +++ b/python/sglang/srt/mem_cache/allocator/token.py @@ -22,7 +22,7 @@ from sglang.srt.mem_cache.allocator.base import BaseTokenToKVPoolAllocator if TYPE_CHECKING: - from sglang.srt.mem_cache.memory_pool import KVCache + from sglang.srt.mem_cache.pool.base import KVCache class TokenToKVPoolAllocator(BaseTokenToKVPoolAllocator): diff --git a/python/sglang/srt/mem_cache/base_swa_memory_pool.py b/python/sglang/srt/mem_cache/base_swa_memory_pool.py deleted file mode 100644 index 2b44bf942e0e..000000000000 --- a/python/sglang/srt/mem_cache/base_swa_memory_pool.py +++ /dev/null @@ -1,32 +0,0 @@ -import abc -from typing import List, Optional, Tuple - -import torch - -from sglang.srt.mem_cache.memory_pool import KVCache - - -class BaseSWAKVPool(KVCache): - """ABC for SWA-like KV pools. - - Subclasses expose a `swa_kv_pool` sub-pool plus a full -> swa index - mapping. Used by `SWATokenToKVPoolAllocator` and the disagg paths to - handle SWA state separately from the full KV state. - """ - - swa_kv_pool: KVCache - # Set when SWA KV is a per-request ring of this many tokens (addressed by - # req_pool_idx) rather than a paged token pool; SWA is then not budgeted per token. - swa_req_ring_size: Optional[int] = None - - @abc.abstractmethod - def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None: - raise NotImplementedError() - - @abc.abstractmethod - def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor) -> torch.Tensor: - raise NotImplementedError() - - @abc.abstractmethod - def get_state_buf_infos(self) -> Tuple[List[int], List[int], List[int]]: - raise NotImplementedError() diff --git a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py index ddc7c7ecafd3..bc1867befd23 100644 --- a/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py +++ b/python/sglang/srt/mem_cache/deepseek_v4_memory_pool.py @@ -18,9 +18,8 @@ from sglang.kernels.ops.attention.dsv4.index_buf_accessor import NopeFp8RopeBf16Pack from sglang.srt.constants import GPU_MEMORY_TYPE_KV_CACHE from sglang.srt.environ import envs -from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.deepseek_v4_compress_state import CompressStatePool -from sglang.srt.mem_cache.memory_pool import KVCache +from sglang.srt.mem_cache.pool.base import BaseSWAKVPool, KVCache from sglang.srt.runtime_context import get_exec, get_spec from sglang.srt.utils import ceil_div, is_hip diff --git a/python/sglang/srt/mem_cache/dsa_cache_layer_split.py b/python/sglang/srt/mem_cache/dsa_cache_layer_split.py index 0f387edae993..13690f21db1f 100644 --- a/python/sglang/srt/mem_cache/dsa_cache_layer_split.py +++ b/python/sglang/srt/mem_cache/dsa_cache_layer_split.py @@ -44,6 +44,8 @@ RadixAttention, get_tensor_size_bytes, maybe_detect_oob, +) +from sglang.srt.mem_cache.pool.base import ( unwrap_write_loc, ) from sglang.srt.runtime_context import get_parallel diff --git a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py index 2d9e5ad77847..8f84218fbc87 100644 --- a/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py +++ b/python/sglang/srt/mem_cache/hybrid_cache/hybrid_pool_assembler.py @@ -1216,7 +1216,7 @@ def build_hicache_draft_sidecars( tree_cache: Any, ) -> tuple[list[SidecarPoolSpec], list[PoolEntry]]: """Compose the full and SWA draft-sidecar paths.""" - from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool + from sglang.srt.mem_cache.pool.base import BaseSWAKVPool assert len(draft_device_pools) == 1 draft_kv_pool = draft_device_pools[0] diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py index 3d6c7c4a8fe1..989ccb2175b3 100644 --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py @@ -62,7 +62,6 @@ DSATokenToKVPool, HybridLinearKVPool, HybridReqToTokenPool, - KVCache, MHATokenToKVPool, MHATokenToKVPoolFP4, MHATokenToKVPoolMXFP8, @@ -73,6 +72,9 @@ PageMajorMHATokenToKVPool, ReqToTokenPool, ) +from sglang.srt.mem_cache.pool.base import ( + KVCache, +) from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.platforms import current_platform from sglang.srt.runtime_context import ( diff --git a/python/sglang/srt/mem_cache/kv_index_translator.py b/python/sglang/srt/mem_cache/kv_index_translator.py index 2e1b970680fa..2d579f0acc92 100644 --- a/python/sglang/srt/mem_cache/kv_index_translator.py +++ b/python/sglang/srt/mem_cache/kv_index_translator.py @@ -70,7 +70,7 @@ from sglang.srt.mem_cache.allocator.unified_mamba import ( UnifiedMambaTokenToKVPoolAllocator, ) -from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool +from sglang.srt.mem_cache.pool.base import BaseSWAKVPool from sglang.srt.runtime_context import get_parallel diff --git a/python/sglang/srt/mem_cache/kv_vmm_backing.py b/python/sglang/srt/mem_cache/kv_vmm_backing.py index 88f626ece6a8..1b6ca908ef1f 100644 --- a/python/sglang/srt/mem_cache/kv_vmm_backing.py +++ b/python/sglang/srt/mem_cache/kv_vmm_backing.py @@ -16,7 +16,7 @@ ) if TYPE_CHECKING: - from sglang.srt.mem_cache.memory_pool import KvBufferDesc + from sglang.srt.mem_cache.pool.base import KvBufferDesc logger = logging.getLogger(__name__) diff --git a/python/sglang/srt/mem_cache/memory_pool.py b/python/sglang/srt/mem_cache/memory_pool.py index 0b23b915629d..7ca365696de7 100644 --- a/python/sglang/srt/mem_cache/memory_pool.py +++ b/python/sglang/srt/mem_cache/memory_pool.py @@ -22,7 +22,6 @@ from __future__ import annotations -import abc import copy import dataclasses import logging @@ -63,6 +62,12 @@ build_page_major_mamba_views, mamba_entry_bytes, ) +from sglang.srt.mem_cache.pool.base import ( + GB, + KvBufferDesc, + KVCache, + unwrap_write_loc, +) from sglang.srt.mem_cache.utils import ( get_mla_kv_buffer_triton, maybe_init_custom_mem_pool, @@ -102,7 +107,6 @@ # re-enable for debugging. _MAMBA_DEBUG_ASSERTS = os.environ.get("SGLANG_MAMBA_DEBUG_ASSERTS", "0") == "1" -GB = 1024 * 1024 * 1024 _is_cuda = is_cuda() _is_npu = is_npu() _is_cpu = is_cpu() @@ -1718,227 +1722,6 @@ def clear(self): self.req_index_to_mamba_ping_pong_track_buffer_mapping.zero_() -@dataclass -class KVWriteLoc: - """Write target(s) for ``KVCache.set_kv_buffer``. - - All location info lives here (in the attention metadata), NOT in the pool: - - ``loc``: the generic per-token write location (``out_cache_loc``). - KERNEL-FACING on every pool: physical by allocation on non-unified - pools, rebound at ForwardBatch construction (``rebind_write_loc``) on - the unified pool. - - ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None`` - otherwise); under the unified pool the translator derives it from the - same rebound loc (``sliding_window_write_loc_for``). - - ``full_loc``: OPTIONAL full-attention-sub-pool location. Since the - construction-time rebind it is the SAME id space as ``loc``, so pools - fall back to ``loc`` when it is ``None`` -- only triton's captured path - still passes its capture-stable - ``ForwardMetadata.out_cache_loc_full_physical`` buffer here (a - same-space alias slated for collapse). - - ``swa_loc`` and ``full_loc`` are the parallel pair (each a pre-resolved - loc into its sub-pool, mirroring ``swa_kv_pool`` / ``full_kv_pool``); - ``loc`` is the generic fallback. Bundling them lets a backend issue one - ``set_kv_buffer`` call regardless of pool type. - """ - - loc: torch.Tensor - swa_loc: Optional[torch.Tensor] = None - full_loc: Optional[torch.Tensor] = None - - def __post_init__(self): - # swa_loc / full_loc are resolved once at metadata-init from the full - # (padded) out_cache_loc; piecewise/DP-padded paths later narrow loc per - # layer, so slice these pre-resolved locs to match (same per-token order). - if self.swa_loc is not None and self.swa_loc.shape[0] != self.loc.shape[0]: - self.swa_loc = self.swa_loc[: self.loc.shape[0]] - if self.full_loc is not None and self.full_loc.shape[0] != self.loc.shape[0]: - self.full_loc = self.full_loc[: self.loc.shape[0]] - - -def unwrap_write_loc(loc_info): - """Return ``(loc, swa_loc, full_loc)`` from a ``KVWriteLoc`` or a bare loc.""" - if isinstance(loc_info, KVWriteLoc): - return loc_info.loc, loc_info.swa_loc, loc_info.full_loc - return loc_info, None, None - - -class KvBufferDesc: - """Byte-span math for one KV buffer laid out as rows of ``row_bytes`` holding - ``tokens_per_row`` tokens each (a row = one token slot, or one whole page).""" - - __slots__ = ("name", "shape", "row_bytes", "tokens_per_row") - - def __init__(self, name: str, shape: tuple, *, row_bytes: int, tokens_per_row: int): - self.name = name - self.shape = tuple(shape) - self.row_bytes = int(row_bytes) - self.tokens_per_row = int(tokens_per_row) - - def _rows(self, num_tokens: int) -> int: - n = max(int(num_tokens), 0) - return (n + self.tokens_per_row - 1) // self.tokens_per_row - - def reserved_span_bytes(self, itemsize: int) -> int: - """Full upper-bound byte size of the buffer (its whole tensor).""" - return math.prod(self.shape) * itemsize - - def prefix_span_bytes(self, num_tokens: int, page_size: int) -> int: - """Bytes to back to make the first ``num_tokens`` tokens usable.""" - return self._rows(num_tokens) * self.row_bytes - - def final_span_bytes(self, num_tokens: int, page_size: int) -> int: - """Bytes of the final advertised span (adds the padded page). CEIL, not floor: - an unaligned count must still cover its partial last page (e.g. n=17, page=16 - -> 3 pages, not 2).""" - return self._rows(max(int(num_tokens), 0) + page_size) * self.row_bytes - - def item_len_bytes(self, page_size: int) -> int: - """Per-page transfer chunk (one page's worth of this buffer).""" - return (page_size // self.tokens_per_row) * self.row_bytes - - -class KVCache(abc.ABC): - layer_shard_enabled: bool = False - post_capture_active: bool = False - # Whether get_cpu_copy/load_cpu_copy carry the recurrent state. False when the - # state lives on the request pool instead, and the caller has to move it. - cpu_copy_carries_mamba: bool = False - - @abc.abstractmethod - def __init__( - self, - size: int, - page_size: int, - dtype: torch.dtype, - layer_num: int, - device: str, - enable_memory_saver: bool, - start_layer: Optional[int] = None, - end_layer: Optional[int] = None, - allocation_label: Optional[str] = None, - ): - self.size = size - self.page_size = page_size - # Row-blocks one page holds in this pool's kernel-facing id space; >1 - # only for the unified pool's per-layer views, and then a write loc must - # have been translated into that space first. - self.kernel_page_blocks = 1 - self.dtype = dtype - self.device = device - if dtype in (torch.float8_e5m2, torch.float8_e4m3fn, torch.float8_e4m3fnuz): - # NOTE: Store as torch.uint8 because Tensor.index_put is not implemented for torch.float8_e5m2 - self.store_dtype = torch.uint8 - else: - self.store_dtype = dtype - self.layer_num = layer_num - self.start_layer = start_layer or 0 - self.end_layer = end_layer or layer_num - 1 - self.allocation_label = allocation_label - self.memory_saver_adapter = TorchMemorySaverAdapter.create( - enable=enable_memory_saver - ) - self.mem_usage = 0 - - # used for chunked cpu-offloading - self.cpu_offloading_chunk_size = 8192 - - # default state for optional layer-wise transfer control - self.layer_transfer_counter = None - - # for disagg with nvlink - self.enable_custom_mem_pool, self.custom_mem_pool, _ = ( - maybe_init_custom_mem_pool(device=self.device) - ) - - def _finalize_allocation_log(self, num_tokens: int): - """Common logging and mem_usage computation for KV cache allocation. - Supports both tuple (K, V) size returns and single KV size returns. - """ - cache_name = ( - f"{self.allocation_label} KV Cache" - if self.allocation_label is not None - else "KV Cache" - ) - kv_size_bytes = self.get_kv_size_bytes() - if isinstance(kv_size_bytes, tuple): - k_size, v_size = kv_size_bytes - k_size_GB = k_size / GB - v_size_GB = v_size / GB - logger.info( - f"{cache_name} {'VA upper bound' if self.post_capture_active else 'is allocated'}. dtype: {self.dtype}, " - f"#tokens: {num_tokens}, K size: {k_size_GB:.2f} GB, " - f"V size: {v_size_GB:.2f} GB" - ) - self.mem_usage = k_size_GB + v_size_GB - else: - kv_size_GB = kv_size_bytes / GB - logger.info( - f"{cache_name} {'VA upper bound' if self.post_capture_active else 'is allocated'}. dtype: {self.dtype}, " - f"#tokens: {num_tokens}, KV size: {kv_size_GB:.2f} GB" - ) - self.mem_usage = kv_size_GB - - def get_kv_buffer_shape(self) -> Tuple[torch.Size, torch.Size]: - k_buffer, v_buffer = self.get_kv_buffer(self.start_layer) - return k_buffer.shape, v_buffer.shape - - @abc.abstractmethod - def get_key_buffer(self, layer_id: int) -> torch.Tensor: - raise NotImplementedError() - - @abc.abstractmethod - def get_value_buffer(self, layer_id: int) -> torch.Tensor: - raise NotImplementedError() - - @abc.abstractmethod - def get_kv_buffer(self, layer_id: int) -> Tuple[torch.Tensor, torch.Tensor]: - raise NotImplementedError() - - @abc.abstractmethod - def set_kv_buffer( - self, - layer: RadixAttention, - loc: torch.Tensor, - cache_k: torch.Tensor, - cache_v: torch.Tensor, - ) -> None: - raise NotImplementedError() - - def register_layer_transfer_counter(self, layer_transfer_counter: LayerDoneCounter): - self.layer_transfer_counter = layer_transfer_counter - - def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None): - raise NotImplementedError() - - def load_cpu_copy( - self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None - ): - raise NotImplementedError() - - def get_kv_cache_quant_method(self) -> Any: - """Return the concrete KV quant method, unwrapping composite KV pools.""" - fallback = None - for pool in ( - self, - getattr(self, "full_kv_pool", None), - getattr(self, "swa_kv_pool", None), - ): - if pool is None: - continue - quant_method = getattr(pool, "quant_method", None) - if quant_method is None: - continue - if getattr(quant_method, "name", None) != "unquantized": - return quant_method - fallback = quant_method - return fallback - - def maybe_get_custom_mem_pool(self): - return self.custom_mem_pool - - class MHATokenToKVPool(KVCache): def __init__( self, diff --git a/python/sglang/srt/mem_cache/pool/__init__.py b/python/sglang/srt/mem_cache/pool/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/sglang/srt/mem_cache/pool/base.py b/python/sglang/srt/mem_cache/pool/base.py new file mode 100644 index 000000000000..18e2f254ca88 --- /dev/null +++ b/python/sglang/srt/mem_cache/pool/base.py @@ -0,0 +1,271 @@ +"""Device-side KV cache abstractions: the write-location bundle, the byte-span +descriptor, and the ``KVCache`` / ``BaseSWAKVPool`` base classes every device pool +under ``mem_cache/pool/`` derives from.""" + +from __future__ import annotations + +import abc +import logging +import math +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, List, Optional, Tuple + +import torch + +from sglang.srt.mem_cache.utils import maybe_init_custom_mem_pool +from sglang.srt.utils.torch_memory_saver_adapter import TorchMemorySaverAdapter + +if TYPE_CHECKING: + from sglang.srt.layers.radix_attention import RadixAttention + from sglang.srt.managers.cache_controller import LayerDoneCounter + +logger = logging.getLogger(__name__) + +GB = 1024 * 1024 * 1024 + + +@dataclass +class KVWriteLoc: + """Write target(s) for ``KVCache.set_kv_buffer``. + + All location info lives here (in the attention metadata), NOT in the pool: + - ``loc``: the generic per-token write location (``out_cache_loc``). + KERNEL-FACING on every pool: physical by allocation on non-unified + pools, rebound at ForwardBatch construction (``rebind_write_loc``) on + the unified pool. + - ``swa_loc``: the SWA-sub-pool location for hybrid SWA pools (``None`` + otherwise); under the unified pool the translator derives it from the + same rebound loc (``sliding_window_write_loc_for``). + - ``full_loc``: OPTIONAL full-attention-sub-pool location. Since the + construction-time rebind it is the SAME id space as ``loc``, so pools + fall back to ``loc`` when it is ``None`` -- only triton's captured path + still passes its capture-stable + ``ForwardMetadata.out_cache_loc_full_physical`` buffer here (a + same-space alias slated for collapse). + + ``swa_loc`` and ``full_loc`` are the parallel pair (each a pre-resolved + loc into its sub-pool, mirroring ``swa_kv_pool`` / ``full_kv_pool``); + ``loc`` is the generic fallback. Bundling them lets a backend issue one + ``set_kv_buffer`` call regardless of pool type. + """ + + loc: torch.Tensor + swa_loc: Optional[torch.Tensor] = None + full_loc: Optional[torch.Tensor] = None + + def __post_init__(self): + # swa_loc / full_loc are resolved once at metadata-init from the full + # (padded) out_cache_loc; piecewise/DP-padded paths later narrow loc per + # layer, so slice these pre-resolved locs to match (same per-token order). + if self.swa_loc is not None and self.swa_loc.shape[0] != self.loc.shape[0]: + self.swa_loc = self.swa_loc[: self.loc.shape[0]] + if self.full_loc is not None and self.full_loc.shape[0] != self.loc.shape[0]: + self.full_loc = self.full_loc[: self.loc.shape[0]] + + +def unwrap_write_loc(loc_info): + """Return ``(loc, swa_loc, full_loc)`` from a ``KVWriteLoc`` or a bare loc.""" + if isinstance(loc_info, KVWriteLoc): + return loc_info.loc, loc_info.swa_loc, loc_info.full_loc + return loc_info, None, None + + +class KvBufferDesc: + """Byte-span math for one KV buffer laid out as rows of ``row_bytes`` holding + ``tokens_per_row`` tokens each (a row = one token slot, or one whole page).""" + + __slots__ = ("name", "shape", "row_bytes", "tokens_per_row") + + def __init__(self, name: str, shape: tuple, *, row_bytes: int, tokens_per_row: int): + self.name = name + self.shape = tuple(shape) + self.row_bytes = int(row_bytes) + self.tokens_per_row = int(tokens_per_row) + + def _rows(self, num_tokens: int) -> int: + n = max(int(num_tokens), 0) + return (n + self.tokens_per_row - 1) // self.tokens_per_row + + def reserved_span_bytes(self, itemsize: int) -> int: + """Full upper-bound byte size of the buffer (its whole tensor).""" + return math.prod(self.shape) * itemsize + + def prefix_span_bytes(self, num_tokens: int, page_size: int) -> int: + """Bytes to back to make the first ``num_tokens`` tokens usable.""" + return self._rows(num_tokens) * self.row_bytes + + def final_span_bytes(self, num_tokens: int, page_size: int) -> int: + """Bytes of the final advertised span (adds the padded page). CEIL, not floor: + an unaligned count must still cover its partial last page (e.g. n=17, page=16 + -> 3 pages, not 2).""" + return self._rows(max(int(num_tokens), 0) + page_size) * self.row_bytes + + def item_len_bytes(self, page_size: int) -> int: + """Per-page transfer chunk (one page's worth of this buffer).""" + return (page_size // self.tokens_per_row) * self.row_bytes + + +class KVCache(abc.ABC): + layer_shard_enabled: bool = False + post_capture_active: bool = False + # Whether get_cpu_copy/load_cpu_copy carry the recurrent state. False when the + # state lives on the request pool instead, and the caller has to move it. + cpu_copy_carries_mamba: bool = False + + @abc.abstractmethod + def __init__( + self, + size: int, + page_size: int, + dtype: torch.dtype, + layer_num: int, + device: str, + enable_memory_saver: bool, + start_layer: Optional[int] = None, + end_layer: Optional[int] = None, + allocation_label: Optional[str] = None, + ): + self.size = size + self.page_size = page_size + # Row-blocks one page holds in this pool's kernel-facing id space; >1 + # only for the unified pool's per-layer views, and then a write loc must + # have been translated into that space first. + self.kernel_page_blocks = 1 + self.dtype = dtype + self.device = device + if dtype in (torch.float8_e5m2, torch.float8_e4m3fn, torch.float8_e4m3fnuz): + # NOTE: Store as torch.uint8 because Tensor.index_put is not implemented for torch.float8_e5m2 + self.store_dtype = torch.uint8 + else: + self.store_dtype = dtype + self.layer_num = layer_num + self.start_layer = start_layer or 0 + self.end_layer = end_layer or layer_num - 1 + self.allocation_label = allocation_label + self.memory_saver_adapter = TorchMemorySaverAdapter.create( + enable=enable_memory_saver + ) + self.mem_usage = 0 + + # used for chunked cpu-offloading + self.cpu_offloading_chunk_size = 8192 + + # default state for optional layer-wise transfer control + self.layer_transfer_counter = None + + # for disagg with nvlink + self.enable_custom_mem_pool, self.custom_mem_pool, _ = ( + maybe_init_custom_mem_pool(device=self.device) + ) + + def _finalize_allocation_log(self, num_tokens: int): + """Common logging and mem_usage computation for KV cache allocation. + Supports both tuple (K, V) size returns and single KV size returns. + """ + cache_name = ( + f"{self.allocation_label} KV Cache" + if self.allocation_label is not None + else "KV Cache" + ) + kv_size_bytes = self.get_kv_size_bytes() + if isinstance(kv_size_bytes, tuple): + k_size, v_size = kv_size_bytes + k_size_GB = k_size / GB + v_size_GB = v_size / GB + logger.info( + f"{cache_name} {'VA upper bound' if self.post_capture_active else 'is allocated'}. dtype: {self.dtype}, " + f"#tokens: {num_tokens}, K size: {k_size_GB:.2f} GB, " + f"V size: {v_size_GB:.2f} GB" + ) + self.mem_usage = k_size_GB + v_size_GB + else: + kv_size_GB = kv_size_bytes / GB + logger.info( + f"{cache_name} {'VA upper bound' if self.post_capture_active else 'is allocated'}. dtype: {self.dtype}, " + f"#tokens: {num_tokens}, KV size: {kv_size_GB:.2f} GB" + ) + self.mem_usage = kv_size_GB + + def get_kv_buffer_shape(self) -> Tuple[torch.Size, torch.Size]: + k_buffer, v_buffer = self.get_kv_buffer(self.start_layer) + return k_buffer.shape, v_buffer.shape + + @abc.abstractmethod + def get_key_buffer(self, layer_id: int) -> torch.Tensor: + raise NotImplementedError() + + @abc.abstractmethod + def get_value_buffer(self, layer_id: int) -> torch.Tensor: + raise NotImplementedError() + + @abc.abstractmethod + def get_kv_buffer(self, layer_id: int) -> Tuple[torch.Tensor, torch.Tensor]: + raise NotImplementedError() + + @abc.abstractmethod + def set_kv_buffer( + self, + layer: RadixAttention, + loc: torch.Tensor, + cache_k: torch.Tensor, + cache_v: torch.Tensor, + ) -> None: + raise NotImplementedError() + + def register_layer_transfer_counter(self, layer_transfer_counter: LayerDoneCounter): + self.layer_transfer_counter = layer_transfer_counter + + def get_cpu_copy(self, indices, mamba_indices=None, req_pool_index=None): + raise NotImplementedError() + + def load_cpu_copy( + self, kv_cache_cpu, indices, mamba_indices=None, req_pool_index=None + ): + raise NotImplementedError() + + def get_kv_cache_quant_method(self) -> Any: + """Return the concrete KV quant method, unwrapping composite KV pools.""" + fallback = None + for pool in ( + self, + getattr(self, "full_kv_pool", None), + getattr(self, "swa_kv_pool", None), + ): + if pool is None: + continue + quant_method = getattr(pool, "quant_method", None) + if quant_method is None: + continue + if getattr(quant_method, "name", None) != "unquantized": + return quant_method + fallback = quant_method + return fallback + + def maybe_get_custom_mem_pool(self): + return self.custom_mem_pool + + +class BaseSWAKVPool(KVCache): + """ABC for SWA-like KV pools. + + Subclasses expose a `swa_kv_pool` sub-pool plus a full -> swa index + mapping. Used by `SWATokenToKVPoolAllocator` and the disagg paths to + handle SWA state separately from the full KV state. + """ + + swa_kv_pool: KVCache + # Set when SWA KV is a per-request ring of this many tokens (addressed by + # req_pool_idx) rather than a paged token pool; SWA is then not budgeted per token. + swa_req_ring_size: Optional[int] = None + + @abc.abstractmethod + def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None: + raise NotImplementedError() + + @abc.abstractmethod + def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor) -> torch.Tensor: + raise NotImplementedError() + + @abc.abstractmethod + def get_state_buf_infos(self) -> Tuple[List[int], List[int], List[int]]: + raise NotImplementedError() diff --git a/python/sglang/srt/mem_cache/pool_host/base.py b/python/sglang/srt/mem_cache/pool_host/base.py index fa06cbf5b1dd..a3b05bbb563b 100644 --- a/python/sglang/srt/mem_cache/pool_host/base.py +++ b/python/sglang/srt/mem_cache/pool_host/base.py @@ -10,7 +10,7 @@ import torch from sglang.srt.distributed.parallel_state import get_world_group -from sglang.srt.mem_cache.memory_pool import KVCache +from sglang.srt.mem_cache.pool.base import KVCache from sglang.srt.mem_cache.pool_host.common import ( _cuda_host_unregister, get_allocator_from_storage, diff --git a/python/sglang/srt/mem_cache/sparsity/core/sparse_coordinator.py b/python/sglang/srt/mem_cache/sparsity/core/sparse_coordinator.py index 993b4a16b075..dca4d458bd0c 100644 --- a/python/sglang/srt/mem_cache/sparsity/core/sparse_coordinator.py +++ b/python/sglang/srt/mem_cache/sparsity/core/sparse_coordinator.py @@ -4,7 +4,8 @@ import torch -from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool +from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.pool.base import KVCache from sglang.srt.mem_cache.sparsity.algorithms.base_algorithm import BaseSparseAlgorithm from sglang.srt.mem_cache.sparsity.backend.backend_adaptor import BackendAdaptor diff --git a/python/sglang/srt/mem_cache/swa_memory_pool.py b/python/sglang/srt/mem_cache/swa_memory_pool.py index 1206168478dd..eea9cf66a2a0 100644 --- a/python/sglang/srt/mem_cache/swa_memory_pool.py +++ b/python/sglang/srt/mem_cache/swa_memory_pool.py @@ -4,12 +4,14 @@ import torch from sglang.srt.layers.radix_attention import RadixAttention -from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool from sglang.srt.mem_cache.memory_pool import ( DSATokenToKVPool, - KVCache, MHATokenToKVPool, MLATokenToKVPool, +) +from sglang.srt.mem_cache.pool.base import ( + BaseSWAKVPool, + KVCache, unwrap_write_loc, ) from sglang.srt.mem_cache.utils import maybe_init_custom_mem_pool diff --git a/python/sglang/srt/mem_cache/unified_memory_pool.py b/python/sglang/srt/mem_cache/unified_memory_pool.py index 8810e5fbaaf0..ec7d1999d9e2 100644 --- a/python/sglang/srt/mem_cache/unified_memory_pool.py +++ b/python/sglang/srt/mem_cache/unified_memory_pool.py @@ -47,6 +47,8 @@ MambaPool, MHATokenToKVPool, MLATokenToKVPool, +) +from sglang.srt.mem_cache.pool.base import ( unwrap_write_loc, ) from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool diff --git a/python/sglang/srt/model_executor/forward_context.py b/python/sglang/srt/model_executor/forward_context.py index eb84508dfe94..a9efc5034cc8 100644 --- a/python/sglang/srt/model_executor/forward_context.py +++ b/python/sglang/srt/model_executor/forward_context.py @@ -28,7 +28,8 @@ if TYPE_CHECKING: from sglang.srt.layers.attention.base_attn_backend import AttentionBackend - from sglang.srt.mem_cache.memory_pool import KVCache, ReqToTokenPool + from sglang.srt.mem_cache.memory_pool import ReqToTokenPool +from sglang.srt.mem_cache.pool.base import KVCache @dataclass(frozen=True, slots=True) diff --git a/python/sglang/srt/models/dots3_common/modeling.py b/python/sglang/srt/models/dots3_common/modeling.py index ad9ce1de687d..d68ca86f8649 100644 --- a/python/sglang/srt/models/dots3_common/modeling.py +++ b/python/sglang/srt/models/dots3_common/modeling.py @@ -102,7 +102,7 @@ MultimodalDataItem, MultimodalInputs, ) -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.model_executor.forward_batch_info import ForwardBatch, PPProxyTensors from sglang.srt.model_executor.forward_context import ( get_attn_backend, diff --git a/python/sglang/srt/models/gemma4_mtp.py b/python/sglang/srt/models/gemma4_mtp.py index 3978a1be17ed..175a3300f283 100644 --- a/python/sglang/srt/models/gemma4_mtp.py +++ b/python/sglang/srt/models/gemma4_mtp.py @@ -29,7 +29,7 @@ LogitsProcessorOutput, ) from sglang.srt.layers.quantization.base_config import QuantizationConfig -from sglang.srt.mem_cache.memory_pool import KVCache +from sglang.srt.mem_cache.pool.base import KVCache from sglang.srt.model_executor.forward_batch_info import ForwardBatch from sglang.srt.models.gemma4_causal import Gemma4ForCausalLM, Gemma4TextModel from sglang.srt.speculative.frozen_kv_mtp_info import FrozenKVMTPContext diff --git a/python/sglang/srt/speculative/frozen_kv_mtp_info.py b/python/sglang/srt/speculative/frozen_kv_mtp_info.py index dc9cc184d9eb..c9d941b538e9 100644 --- a/python/sglang/srt/speculative/frozen_kv_mtp_info.py +++ b/python/sglang/srt/speculative/frozen_kv_mtp_info.py @@ -16,7 +16,7 @@ from dataclasses import dataclass from typing import Dict -from sglang.srt.mem_cache.memory_pool import KVCache +from sglang.srt.mem_cache.pool.base import KVCache from sglang.srt.speculative.eagle_info import ( EagleDraftInput, EagleVerifyInput, diff --git a/test/registered/attention/unittests/swa/test_swa_out_cache_loc.py b/test/registered/attention/unittests/swa/test_swa_out_cache_loc.py index 0d0317c5a5b5..31f7bae1fbed 100644 --- a/test/registered/attention/unittests/swa/test_swa_out_cache_loc.py +++ b/test/registered/attention/unittests/swa/test_swa_out_cache_loc.py @@ -18,7 +18,8 @@ from sglang.srt.layers.attention.dots_hybrid_backend import DotsSWAMLAAttnBackend from sglang.srt.layers.attention.flashattention_backend import FlashAttentionBackend -from sglang.srt.mem_cache.memory_pool import KVWriteLoc, MLATokenToKVPool +from sglang.srt.mem_cache.memory_pool import MLATokenToKVPool +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.model_executor.forward_batch_info import ForwardMode from sglang.test.ci.ci_register import register_cpu_ci diff --git a/test/registered/cp/test_cp_strategy_unit.py b/test/registered/cp/test_cp_strategy_unit.py index 1bc6a537d62d..c3a5cb3db71e 100644 --- a/test/registered/cp/test_cp_strategy_unit.py +++ b/test/registered/cp/test_cp_strategy_unit.py @@ -28,7 +28,7 @@ prepare_cp_forward, ) from sglang.srt.layers.cp.zigzag import ZigzagCPStrategy -from sglang.srt.mem_cache.memory_pool import KVWriteLoc +from sglang.srt.mem_cache.pool.base import KVWriteLoc from sglang.srt.model_executor.cuda_graph_config import Backend from sglang.srt.model_executor.forward_batch_info import ( CaptureHiddenMode, diff --git a/test/registered/unit/mem_cache/test_full_loc_fast_path.py b/test/registered/unit/mem_cache/test_full_loc_fast_path.py index 8c301633e5d9..8e5c59354ce8 100644 --- a/test/registered/unit/mem_cache/test_full_loc_fast_path.py +++ b/test/registered/unit/mem_cache/test_full_loc_fast_path.py @@ -31,7 +31,7 @@ def _loc_info(virtual_loc, swa_phys=None, full_phys=None): - from sglang.srt.mem_cache.memory_pool import KVWriteLoc + from sglang.srt.mem_cache.pool.base import KVWriteLoc return KVWriteLoc(virtual_loc, swa_phys, full_phys) diff --git a/test/registered/unit/spec/test_resolve_swa_kv_pool.py b/test/registered/unit/spec/test_resolve_swa_kv_pool.py index 57e40d633134..70dc887e1588 100644 --- a/test/registered/unit/spec/test_resolve_swa_kv_pool.py +++ b/test/registered/unit/spec/test_resolve_swa_kv_pool.py @@ -5,7 +5,7 @@ from sglang.srt.layers.attention.flashinfer_backend import FlashInferAttnBackend from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend -from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool +from sglang.srt.mem_cache.pool.base import BaseSWAKVPool from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool from sglang.srt.speculative.spec_info import SpeculativeAlgorithm from sglang.test.ci.ci_register import register_amd_ci, register_cuda_ci