Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions python/sglang/srt/managers/schedule_batch.py
Original file line number Diff line number Diff line change
Expand Up @@ -3472,6 +3472,9 @@ def mamba_lazy_prealloc_at_boundary(self, mamba_track_interval: int):
if new_slot is not None:
pool.set_mamba_ping_pong_slot(req, other_idx, new_slot[0])
req.kv.mamba_next_track_idx = other_idx
else:
# The in-flight write invalidates the keep slot's old checkpoint.
req.kv.mamba_last_track_seqlen = None

def mamba_lazy_spec_prepare(self, mamba_track_interval: int, max_draft_tokens: int):
"""Lazy-mode spec counterpart of mamba_lazy_prealloc_at_boundary.
Expand Down Expand Up @@ -3508,6 +3511,8 @@ def mamba_lazy_spec_prepare(self, mamba_track_interval: int, max_draft_tokens: i
if new_slot is not None:
pool.set_mamba_ping_pong_slot(req, other_idx, new_slot[0])
has_pending = True
else:
req.kv.mamba_last_track_seqlen = None
# On failure the verify scatters in place into the keep slot.
track_positions.append(
other_idx if has_pending else req.kv.mamba_next_track_idx
Expand Down
137 changes: 137 additions & 0 deletions test/registered/radix_cache/test_mamba_cache_kl.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,137 @@
"""KL regression tests for Mamba prefix-cache correctness.

Keep each scenario in its own class so server configurations and cache-pressure
requirements remain explicit.
"""

import unittest

import numpy as np
import requests
from transformers import AutoConfig

from sglang.test.ci.ci_register import register_cuda_ci
from sglang.test.server_fixtures.default_fixture import DefaultServerBase

register_cuda_ci(est_time=120, stage="extra-a", runner_config="1-gpu-large")


class TestMambaLazyInplaceKL(DefaultServerBase):
"""Guard lazy Mamba checkpoint reuse when the optional track slot is exhausted.

Keep overlap scheduling and decode CUDA graphs enabled. A four-slot pool holds
two cached prefixes, the request's live state, and its keep checkpoint. The
first decode must track in place; delayed prefill processing must not publish
that buffer under the old checkpoint length.

Compare full-vocabulary next-token distributions at the SAME 255-token prompt,
before and after a 32-token decode seeds the cache. Comparing later free-running
tokens would confound cache corruption with different generated histories.

GB300 / Qwen3.5-0.8B: reverting the checkpoint invalidation gives KL=0.649448
and a stale 192-token cache hit; with the fix KL=0 and the safe hit is 128 tokens.
"""

model = "Qwen/Qwen3.5-0.8B"
server_env = {
"SGLANG_OPT_MAMBA_SKIP_DECODE_LOCK": "0",
"SGLANG_TEST_MAMBA_LAZY_ALLOC_FAIL": "0",
}
other_args = [
"--skip-tokenizer-init",
"--mamba-radix-cache-strategy",
"extra_buffer_lazy",
"--mamba-track-interval",
"128",
"--max-mamba-cache-size",
"4",
"--max-running-requests",
"1",
"--max-total-tokens",
"8192",
"--mem-fraction-static",
"0.3",
"--cuda-graph-max-bs-decode",
"1",
"--cuda-graph-backend-prefill",
"disabled",
"--chunked-prefill-size",
"2048",
"--attention-backend",
"triton",
"--linear-attn-backend",
"triton",
"--mamba-backend",
"triton",
"--random-seed",
"42",
]

def _generate(self, input_ids, max_new_tokens, *, score=False):
payload = {
"input_ids": input_ids,
"sampling_params": {
"temperature": 0,
"max_new_tokens": max_new_tokens,
"ignore_eos": True,
},
"return_logprob": True,
"logprob_start_len": -1,
}
if score:
payload["token_ids_logprob"] = self.vocab_ids
response = requests.post(self.base_url + "/generate", json=payload, timeout=120)
response.raise_for_status()
result = response.json()
self.assertEqual(result["meta_info"]["completion_tokens"], max_new_tokens)
return result

def _warm_prefixes(self):
response = requests.post(self.base_url + "/flush_cache", timeout=30)
response.raise_for_status()
self._generate([44] * 128, 1)
self._generate([42] * 128, 1)

def _distribution(self, result):
rows = result["meta_info"]["output_token_ids_logprobs"]
self.assertEqual(len(rows), 1)
self.assertEqual([row[1] for row in rows[0]], self.vocab_ids)
logprobs = np.asarray([row[0] for row in rows[0]], dtype=np.float64)
self.assertTrue(np.isfinite(logprobs).all())
self.assertAlmostEqual(float(np.exp(logprobs).sum()), 1.0, places=4)
# Remove log-softmax rounding error before computing the exact KL.
return logprobs - np.logaddexp.reduce(logprobs)

def test_reuse_distribution_after_inplace_tracking(self):
config = AutoConfig.from_pretrained(self.model)
self.vocab_ids = list(range(config.get_text_config().vocab_size))
prompt = [42] * 128 + [43] * 127

# A single output token does not launch the boundary-crossing decode.
self._warm_prefixes()
reference = self._generate(prompt, 1, score=True)
self.assertEqual(reference["meta_info"]["cached_tokens"], 128)
reference_logprobs = self._distribution(reference)

self._warm_prefixes()
seed = self._generate(prompt, 32)
self.assertEqual(seed["meta_info"]["cached_tokens"], 128)
reused = self._generate(prompt, 1, score=True)
self.assertGreaterEqual(reused["meta_info"]["cached_tokens"], 128)
reused_logprobs = self._distribution(reused)

kl = float(
np.sum(np.exp(reference_logprobs) * (reference_logprobs - reused_logprobs))
)
print(
f"lazy in-place checkpoint KL(reference || reuse)={kl:.9g}, "
f"cached_tokens={reused['meta_info']['cached_tokens']}",
flush=True,
)
self.assertGreaterEqual(kl, -1e-12)
# Leave room for numerical noise while staying far below the broken run.
self.assertLess(kl, 1e-4, "Cache reuse changed the next-token distribution")


if __name__ == "__main__":
unittest.main()
Loading