Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
025c5cb
refactor: hoist staged admission deferral and Mamba request-slot allo…
alphabetc1 Sep 30, 2026
62a02a7
fix: charge staged and host-hit Mamba node slots in prefill admission
alphabetc1 Sep 30, 2026
e840bf6
feat: support Mamba in HiCache buffer-only host memory mode
alphabetc1 Sep 30, 2026
5706ad2
test: add buffer-only Mamba storage-hit KL test on Inkling
alphabetc1 Sep 30, 2026
5a0dad6
chore: merge main and resolve buffer-mode storage-hit attribution con…
alphabetc1 Oct 1, 2026
b4777c4
fix: resolve main conflicts in Mamba buffer-only cache
alphabetc1 Oct 4, 2026
b73fbdc
Merge branch 'main' into buffer-only-mamba
alphabetc1 Oct 5, 2026
94507fb
chore: merge main and drop the Mamba write-path keys superseded by pe…
alphabetc1 Oct 8, 2026
1f0cb69
fix: charge the host-hit Mamba node slot in prefill admission
alphabetc1 Oct 8, 2026
4cf8029
feat: support Mamba in HiCache buffer-only host memory mode
alphabetc1 Oct 8, 2026
e24b585
chore: split the admission fix out and move the Mamba load-back into …
alphabetc1 Oct 8, 2026
f482fb9
feat: support Mamba in HiCache buffer-only host memory mode
alphabetc1 Oct 9, 2026
35f14b1
refactor: drop the admission fix and the unused Mamba host-pool fences
alphabetc1 Oct 9, 2026
299d6ad
test: fold the buffer-only storage-hit assertions
alphabetc1 Oct 9, 2026
2845fd8
test: drop the Mamba buffer-only fence test
alphabetc1 Oct 9, 2026
5ac0c61
refactor: clarify component buffer load-back contexts
alphabetc1 Oct 9, 2026
b884b8b
fix: reserve Mamba checkpoint slots before load-back admission
alphabetc1 Oct 9, 2026
fccd048
refactor: preserve existing load_xfers naming
alphabetc1 Oct 9, 2026
79cf3d1
fix: resolve main conflicts in Mamba buffer-only load-back
alphabetc1 Oct 10, 2026
a7cd04e
Merge branch 'main' into buffer-only-mamba
alphabetc1 Oct 10, 2026
d3c178d
Merge branch 'main' into buffer-only-mamba
alphabetc1 Oct 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 37 additions & 29 deletions python/sglang/srt/managers/schedule_policy.py
Original file line number Diff line number Diff line change
Expand Up @@ -704,7 +704,7 @@ def __init__(
self.token_to_kv_pool_allocator.mamba_slot_full_token_cost()
)

# `mamba_gap_reserve` is charged to `rem_total_tokens`, which INCLUDES
# The Mamba slots' gap cost is charged to `rem_total_tokens`, which INCLUDES
# `full_evictable_size()` — but `alloc_req_slots` can only recover
# MAMBA-recoverable bytes for a mamba slot (shared gap + peer holes +
# mamba-evictable radix), NOT full-evictable. Gate new mamba slots on
Expand Down Expand Up @@ -829,20 +829,19 @@ def _check_prefill_budget(
chunk_limit=self.rem_chunk_tokens,
)

def _mamba_gap_budget_for_req(self, req: Req) -> int:
"""Shared-gap reservation (full-token-equivalents) for a request's new
mamba state. Charged only on the SHARED Mamba pool (`_mamba_slot_cost > 0`)
and only when the req has no state yet (`mamba_pool_idx is None`, mirroring
`HybridReqToTokenPool.alloc`); 0 keeps baseline / SWA / non-Mamba unchanged.
def _mamba_slots_for_req(self, req: Req) -> int:
"""Mamba state slots a request will consume on the SHARED Mamba pool
(`_mamba_slot_cost > 0`); 0 keeps baseline / SWA / non-Mamba unchanged.

Conservative by design (`_mamba_slot_cost` rounds UP). Does NOT reserve
radix COW headroom or locked-but-evictable bytes — that residual is
backstopped by the fail-loud RuntimeError in `alloc_req_slots`. FIXME: if
over-admission crashes under pressure, make this more conservative (e.g.
multiply by `MAMBA_STATE_PER_REQ_PREFIX_CACHE`)."""
if self._mamba_slot_cost and not req.kv.holds_mamba:
return self._mamba_slot_cost
return 0
if not self._mamba_slot_cost:
return 0
slots = 0 if req.kv.holds_mamba else 1
return slots + req.mamba_host_hit_length

def ceil_paged_tokens(self, tokens: int) -> int:
return -(-tokens // self.page_size) * self.page_size
Expand Down Expand Up @@ -874,7 +873,7 @@ def _update_prefill_budget(
extend_input_len: int,
max_new_tokens: int,
retracted_stain: bool,
mamba_gap_reserve: int = 0,
mamba_slots: int = 0,
is_chunked_continuation: bool = False,
compute_charge: Optional[int] = None,
):
Expand All @@ -900,15 +899,15 @@ def _update_prefill_budget(
max_new_tokens,
# reserve() already charges one page; add the remaining shard pages.
extra_tokens=(
mamba_gap_reserve + self.per_req_token_overhead - self.page_size
mamba_slots * self._mamba_slot_cost
+ self.per_req_token_overhead
- self.page_size
),
chunk_limit=self.rem_chunk_tokens,
is_chunked_continuation=is_chunked_continuation,
)
# The new mamba slot also consumes one mamba-recoverable slot (gated
# separately so full_evictable can't cover it — see __init__).
if mamba_gap_reserve and self.rem_mamba_slots is not None:
self.rem_mamba_slots -= 1
if mamba_slots and self.rem_mamba_slots is not None:
self.rem_mamba_slots -= mamba_slots
self.rem_input_tokens -= compute_charge

if self.dllm_config is not None:
Expand Down Expand Up @@ -1018,7 +1017,7 @@ def _add_dllm_req(self, req: Req, prefix_len: int):
trunc_len,
0,
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
mamba_slots=self._mamba_slots_for_req(req),
)
self._account_prefill_cache_admission(req, prefix_len)

Expand Down Expand Up @@ -1101,7 +1100,7 @@ def add_dllm_staging_req(self, req: Req):
req.extend_range.length,
max_new_tokens,
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
mamba_slots=self._mamba_slots_for_req(req),
)

# Return based on remaining token availability
Expand Down Expand Up @@ -1173,7 +1172,7 @@ def add_chunked_req(self, req: Req):
else 0
),
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
mamba_slots=self._mamba_slots_for_req(req),
is_chunked_continuation=True,
compute_charge=req.extend_range.length if self.exact_chunk_fill else None,
)
Expand Down Expand Up @@ -1206,7 +1205,7 @@ def add_one_req_ignore_eos(self, req: Req):
)
# Shared Mamba pool: fold the new mamba state's shared-gap cost into the
# budget gate so admission can't over-commit (0 for baseline / non-Mamba).
paged_input += self._mamba_gap_budget_for_req(req)
paged_input += self._mamba_slots_for_req(req) * self._mamba_slot_cost
fits = self.memory_budget.can_allocate_prefill(
paged_input=paged_input,
extend_input_len=cand_extend_input_len,
Expand Down Expand Up @@ -1308,7 +1307,7 @@ def add_req_state(r, insert_sort=False):
req.extend_range.length,
min(req.sampling_params.max_new_tokens, CLIP_MAX_NEW_TOKENS),
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
mamba_slots=self._mamba_slots_for_req(req),
compute_charge=(
req.extend_range.length if self.exact_chunk_fill else None
),
Expand Down Expand Up @@ -1338,7 +1337,7 @@ def add_req_state(r, insert_sort=False):
trunc_len,
0,
req.retracted_stain,
mamba_gap_reserve=self._mamba_gap_budget_for_req(req),
mamba_slots=self._mamba_slots_for_req(req),
compute_charge=trunc_len if self.exact_chunk_fill else None,
)

Expand All @@ -1353,6 +1352,17 @@ def add_one_req(
if req.sampling_params.ignore_eos and getattr(self.tree_cache, "disable", True):
return self.add_one_req_ignore_eos(req)

# A staged checkpoint needs a node slot as well as the request state.
# Rejection participates in staged lifecycle even before materialization.
mamba_slots = self._mamba_slots_for_req(req)
if self.rem_mamba_slots is not None and mamba_slots > self.rem_mamba_slots:
pipeline = self.tree_cache.buffer_pipeline
if pipeline is not None:
pipeline.defer_staged_admission(req, pool="mamba")
# A dropped hold still has this round's FULL-only splice prefix.
# End the attempt; init_next_round_input rebuilds it next round.
return AddReqResult.NO_TOKEN

# Reserve page_size for page-alignment overhead: the paged allocator may
# consume one extra page per request (see alloc_extend), which
# _update_prefill_budget also deducts.
Expand All @@ -1364,12 +1374,10 @@ def add_one_req(
req.prefix_indices
)
total_tokens = cand_extend_input_len + max_new + self.per_req_token_overhead
# Shared Mamba pool: fold the new mamba state's shared-gap cost into
# Shared Mamba pool: fold the mamba slots' shared-gap cost into
# `total_tokens` so both `rem_total_tokens` gates reflect the joint budget.
# Read before `init_load_back` binds `req.mamba_pool_idx` — after that
# this returns 0, so the debit sites below reuse the value.
mamba_gap_reserve = self._mamba_gap_budget_for_req(req)
total_tokens += mamba_gap_reserve
# Read before `init_load_back`; debit sites below reuse the value.
total_tokens += mamba_slots * self._mamba_slot_cost

# The temporary pin excludes this prefix from the evictable budget.
# Selection itself neither allocates slots nor materializes host hits.
Expand Down Expand Up @@ -1457,7 +1465,7 @@ def add_one_req(
):
return AddReqResult.OTHER

self._commit_prefill_admission(req, admission, mamba_gap_reserve)
self._commit_prefill_admission(req, admission, mamba_slots)

# This verdict controls the next candidate, not the committed request.
return self.budget_state()
Expand Down Expand Up @@ -1546,7 +1554,7 @@ def _select_prefill_admission(
return _PrefillAdmission(prefix_len, extend_len, max_new_tokens, is_chunked)

def _commit_prefill_admission(
self, req: Req, admission: _PrefillAdmission, mamba_gap_reserve: int
self, req: Req, admission: _PrefillAdmission, mamba_slots: int
) -> None:
assert len(req.prefix_indices) == admission.prefix_len
req.set_extend_range(
Expand All @@ -1561,7 +1569,7 @@ def _commit_prefill_admission(
admission.extend_len,
admission.max_new_tokens,
req.retracted_stain,
mamba_gap_reserve=mamba_gap_reserve,
mamba_slots=mamba_slots,
# Compute budgets are billed forward-pass tokens under exact-chunk-fill.
compute_charge=admission.extend_len if self.exact_chunk_fill else None,
)
Expand Down
Loading
Loading