From 2f322de92c2d43787a905a799f0785bef7a8ffa2 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Tue, 13 Oct 2026 14:22:58 -0700 Subject: [PATCH 01/10] [DSv4.1] Cover bounded replay BCG with GSM8K and MMMU-Pro --- .../models/test_deepseek_v41_flash_gb300.py | 56 ++++++++++++------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py index c7f8b409fa57..27347b2d06f5 100644 --- a/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py @@ -1,6 +1,7 @@ """DeepSeek-V4.1-Flash accuracy with DSPARK on four GB300 GPUs. -Run GSM8K and MMLU in TP4+EP4 and DP-attention4 + EP4 configurations. +Run GSM8K and MMMU-Pro in TP4+EP4 and DP-attention4 + EP4 configurations, +plus TP4+EP4 with decoder SWA bounded replay inside the breakable prefill graph. The DP+EP recipe is intentionally exercised rather than skipped: startup or accuracy failures must be visible in CI while this path is being validated. """ @@ -11,7 +12,7 @@ from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.eval_accuracy_kit import GSM8KMixin, _run_accuracy_eval +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin, MMMUProMixin from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -19,23 +20,26 @@ try_cached_model, ) -register_cuda_ci(est_time=1100, stage="base-c", runner_config="4-gpu-gb300") +register_cuda_ci(est_time=1800, stage="base-c", runner_config="4-gpu-gb300") MODEL = "deepseek-ai/DeepSeek-V4.1-Flash" -class DSV41FlashAccuracyMixin(GSM8KMixin): +class DSV41FlashAccuracyMixin(GSM8KMixin, MMMUProMixin): # Use the zero-shot chat evaluator, not the legacy five-shot completion path. gsm8k_backend = "sgl_eval" gsm8k_score_threshold = 0.95 gsm8k_num_examples = 200 gsm8k_num_threads = 64 gsm8k_max_tokens = 4096 - mmlu_score_threshold = 0.90 - mmlu_num_examples = 500 - mmlu_num_threads = 64 - mmlu_max_tokens = 4096 - + mmmu_pro_score_threshold = 0.75 + mmmu_pro_num_examples = 200 + mmmu_pro_num_threads = 64 + mmmu_pro_load_preset_from_model_id = MODEL + + mem_fraction_static = "0.8" + max_running_requests = "64" + graph_args = [] parallel_args = [] server_env = {} @@ -56,16 +60,17 @@ def setUpClass(cls): "--speculative-dspark-block-size", "5", "--mem-fraction-static", - "0.8", + cls.mem_fraction_static, "--cuda-graph-max-bs-decode", "64", "--max-running-requests", - "64", + cls.max_running_requests, "--reasoning-parser", "auto", "--tool-call-parser", "auto", *cls.parallel_args, + *cls.graph_args, ], env=cls.server_env, ) @@ -87,15 +92,8 @@ def test_gsm8k(self): super().test_gsm8k() self.assert_dspark_accepts_draft_tokens() - def test_mmlu(self): - _run_accuracy_eval( - self, - eval_name="mmlu", - score_threshold=self.mmlu_score_threshold, - num_examples=self.mmlu_num_examples, - num_threads=self.mmlu_num_threads, - max_tokens=self.mmlu_max_tokens, - ) + def test_mmmu_pro(self): + super().test_mmmu_pro() self.assert_dspark_accepts_draft_tokens() @@ -122,5 +120,23 @@ class TestDSV41FlashDP4EP4DSpark(DSV41FlashAccuracyMixin, CustomTestCase): ] +class TestDSV41FlashTP4EP4DSparkBoundedReplayBCG(TestDSV41FlashTP4EP4DSpark): + """Exercise bounded late-layer replay inside the breakable prefill graph.""" + + # Leave memory for the prefill graph pool. With 32 requests, the tail + # capacity is 4096 rows, below the largest 8192-token capture shape. + mem_fraction_static = "0.7" + max_running_requests = "32" + graph_args = [ + "--enable-decoder-swa-bounded-replay", + "--cuda-graph-backend-prefill", + "breakable", + "--cuda-graph-max-bs-prefill", + "8192", + "--chunked-prefill-size", + "8192", + ] + + if __name__ == "__main__": unittest.main() From ceabcf64fb58d3f610fc7a67d150f831d2a0d8cf Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Tue, 13 Oct 2026 20:32:05 -0700 Subject: [PATCH 02/10] [DSv4.1] Use sgl-eval 0.1.3 for the Flash model preset --- python/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index c1cff46ed9c4..31d96b5f0dd5 100755 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -188,7 +188,7 @@ test = [ # Pinned: a bump moves scoring for every eval test at once, so re-baseline # MODEL_SCORE_THRESHOLDS in test/registered/eval/test_text_models_gsm8k_eval.py # and the mmlu thresholds of run_eval's other callers before changing it. - "sgl-eval==0.1.2", + "sgl-eval==0.1.3", "sglang[fastokens]", "tabulate", ] From 1944a212c5f6ca81cda3cee2d3b3cbea4389271f Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Wed, 14 Oct 2026 17:31:08 -0700 Subject: [PATCH 03/10] [DSv4.1] Cap MMMU-Pro output at 32768 tokens in GB300 CI --- .../e2e/models/test_deepseek_v41_flash_gb300.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py index 27347b2d06f5..c5198ce34935 100644 --- a/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py @@ -12,7 +12,7 @@ from sglang.srt.utils import kill_process_tree from sglang.test.ci.ci_register import register_cuda_ci -from sglang.test.kits.eval_accuracy_kit import GSM8KMixin, MMMUProMixin +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin, _run_accuracy_eval from sglang.test.test_utils import ( DEFAULT_URL_FOR_TEST, CustomTestCase, @@ -25,7 +25,7 @@ MODEL = "deepseek-ai/DeepSeek-V4.1-Flash" -class DSV41FlashAccuracyMixin(GSM8KMixin, MMMUProMixin): +class DSV41FlashAccuracyMixin(GSM8KMixin, _run_accuracy_eval): # Use the zero-shot chat evaluator, not the legacy five-shot completion path. gsm8k_backend = "sgl_eval" gsm8k_score_threshold = 0.95 @@ -35,6 +35,8 @@ class DSV41FlashAccuracyMixin(GSM8KMixin, MMMUProMixin): mmmu_pro_score_threshold = 0.75 mmmu_pro_num_examples = 200 mmmu_pro_num_threads = 64 + # Bound long reasoning responses consistently across all three CI recipes. + mmmu_pro_max_tokens = 32768 mmmu_pro_load_preset_from_model_id = MODEL mem_fraction_static = "0.8" @@ -93,7 +95,16 @@ def test_gsm8k(self): self.assert_dspark_accepts_draft_tokens() def test_mmmu_pro(self): - super().test_mmmu_pro() + _run_accuracy_eval( + self, + eval_name="mmmu_pro", + score_threshold=self.mmmu_pro_score_threshold, + num_examples=self.mmmu_pro_num_examples, + num_threads=self.mmmu_pro_num_threads, + model=None, + load_preset_from_model_id=self.mmmu_pro_load_preset_from_model_id, + max_tokens=self.mmmu_pro_max_tokens, + ) self.assert_dspark_accepts_draft_tokens() From 961dd5f3418a75eb29b5c289608dcbf7cb8374a3 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Wed, 14 Oct 2026 20:55:28 -0700 Subject: [PATCH 04/10] [DSv4.1] Fix accuracy mixin inheritance --- test/registered/e2e/models/test_deepseek_v41_flash_gb300.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py b/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py index c5198ce34935..c3b0ef769588 100644 --- a/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py +++ b/test/registered/e2e/models/test_deepseek_v41_flash_gb300.py @@ -25,7 +25,7 @@ MODEL = "deepseek-ai/DeepSeek-V4.1-Flash" -class DSV41FlashAccuracyMixin(GSM8KMixin, _run_accuracy_eval): +class DSV41FlashAccuracyMixin(GSM8KMixin): # Use the zero-shot chat evaluator, not the legacy five-shot completion path. gsm8k_backend = "sgl_eval" gsm8k_score_threshold = 0.95 From 64c46d08ca581a415793715defc371b78beab36f Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 15 Oct 2026 12:48:09 -0700 Subject: [PATCH 05/10] [DSv4.1] Fix multimodal token IDs and Engram masking in prefill graphs --- .../runner/prefill_cuda_graph_runner.py | 14 ++ python/sglang/srt/models/deepseek_v4.py | 6 +- .../test_deepseek_v41_bcg_multimodal.py | 120 ++++++++++++++++++ 3 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index b4c1fccd9156..e6ca100e010e 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -2104,6 +2104,20 @@ def _execute_body_capture( full_path = self._is_full_backend def replay_layer_forward(*args, **layer_kwargs): + # The eager wrapper may remap multimodal hash placeholders to model + # token IDs. The captured body must see those IDs as well as the + # composed embeddings (e.g. for visual MoE routing and Engram masks). + # Update only the private static buffer, leaving scheduler IDs intact. + prepared_ids = layer_kwargs.get("input_ids") + if prepared_ids is None and args: + prepared_ids = args[0] + if ( + prepared_ids is not None + and prepared_ids.data_ptr() != static_forward_batch.input_ids.data_ptr() + ): + static_forward_batch.input_ids[: prepared_ids.shape[0]].copy_( + prepared_ids + ) # The captured body graph reads activations from the static # input_embeds slot. The outer model.forward (run eagerly) # passes the live embeddings into layer_model.forward as the diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index eaafd01966b2..3aed906c37d4 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -4219,12 +4219,12 @@ def _forward_layers_hc_pre_from_prev( forward_batch, cp_all_tokens=cp_extend, ) - # Only multimodal placeholders become image_token_id, so a text-only - # batch skips this full-residual where. + # Capture the token mask even when the dummy batch has no media: + # later graph replays can contain image tokens. Text rows retain + # the Engram result without depending on Python batch metadata. if ( self.config.model_type == "deepseek_v41" and self.config.vision_n_layers > 0 - and forward_batch.contains_mm_inputs() ): hidden_states = torch.where( (input_ids == self.config.image_token_id)[:, None, None], diff --git a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py new file mode 100644 index 000000000000..022d442822ba --- /dev/null +++ b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py @@ -0,0 +1,120 @@ +"""Regression tests for multimodal IDs and Engram masking during graph replay.""" + +import unittest +from contextlib import nullcontext +from types import SimpleNamespace +from unittest.mock import patch + +import torch + +import sglang.srt.models.deepseek_v4 as dsv4 +from sglang.srt.model_executor.forward_batch_info import ForwardMode +from sglang.srt.model_executor.runner.prefill_cuda_graph_runner import ( + PrefillCudaGraphRunner, +) +from sglang.test.ci.ci_register import register_cpu_ci +from sglang.test.test_utils import CustomTestCase + +register_cpu_ci(est_time=10, suite="base-a-test-cpu") + + +class TestDeepseekV41BCGMultimodal(CustomTestCase): + def test_replay_uses_prepared_image_ids_without_mutating_scheduler_ids(self): + for keyword in (False, True): + with self.subTest(keyword=keyword): + scheduler_ids = torch.tensor([7, dsv4.MM_PAD_SHIFT_VALUE + 1, 9, 0]) + static_ids = scheduler_ids.clone() + image_id = 42 + layer = SimpleNamespace(forward=None) + runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) + runner._is_full_backend = False + runner._qwen_bcg_hc_sidechannel = False + runner._use_draft_input_embeds = False + runner.layer_model = layer + runner.buffer_registry = SimpleNamespace(has_slot=lambda name: False) + runner._prefill_forward_context = lambda *a, **kw: nullcontext() + runner.backend = SimpleNamespace( + replay=lambda *a, **kw: static_ids.clone() + ) + + def outer_forward(ids, positions, batch): + model = SimpleNamespace( + vision=object(), config=SimpleNamespace(image_token_id=image_id) + ) + mapped, _ = ( + dsv4.DeepseekV4ForCausalLM.prepare_language_model_inputs( + model, ids, batch + ) + ) + if keyword: + return layer.forward( + input_ids=mapped, positions=positions, forward_batch=batch + ) + return layer.forward(mapped, positions, batch) + + runner.model_runner = SimpleNamespace( + pp_group=SimpleNamespace(is_first_rank=True), + model=SimpleNamespace(forward=outer_forward), + ) + live = SimpleNamespace(input_ids=scheduler_ids, mm_input_embeds=None) + static = SimpleNamespace( + input_ids=static_ids, + positions=torch.arange(4), + mm_input_embeds=None, + mm_inputs=None, + forward_mode=ForwardMode.EXTEND, + ) + actual = runner._execute_body_capture(live, static, 4, 3, None) + torch.testing.assert_close(actual, torch.tensor([7, image_id, 9, 0])) + self.assertEqual(scheduler_ids[1].item(), dsv4.MM_PAD_SHIFT_VALUE + 1) + + def test_image_engram_mask_does_not_depend_on_capture_batch_mm_metadata(self): + image_id = 42 + ids = torch.tensor([7, image_id, 9]) + hidden = torch.arange(6, dtype=torch.float32).reshape(3, 1, 2) + + class Engram: + layer_hash_index = 0 + + def __call__(self, residual, *args, **kwargs): + return residual + 10 + + layer = SimpleNamespace( + engram=Engram(), + hc_cfg=None, + forward_hc_pre_from_prev=lambda **kw: kw["state"], + ) + model = SimpleNamespace( + pp_group=SimpleNamespace(world_size=1), + engram_hasher=lambda *a: torch.zeros((3, 1), dtype=torch.long), + late_layer_start=None, + start_layer=0, + end_layer=1, + layers=[layer], + config=SimpleNamespace( + model_type="deepseek_v41", vision_n_layers=1, image_token_id=image_id + ), + ) + expected = hidden + 10 + expected[1] = hidden[1] + for contains_mm in (False, True): + batch = SimpleNamespace( + forward_mode=ForwardMode.EXTEND, contains_mm_inputs=lambda: contains_mm + ) + with ( + patch.object(dsv4, "is_cp_active", return_value=False), + patch.object(dsv4, "is_in_breakable_cuda_graph", return_value=False), + patch.object( + dsv4, "get_forward", return_value=SimpleNamespace(sp_active=False) + ), + patch.object(dsv4, "check_cuda_graph_backend", return_value=True), + patch.object(dsv4, "_is_hip", False), + ): + actual, _, _ = dsv4.DeepseekV4Model._forward_layers_hc_pre_from_prev( + model, torch.arange(3), hidden, batch, ids, ids, False, [] + ) + torch.testing.assert_close(actual, expected) + + +if __name__ == "__main__": + unittest.main() From 786470debece0e684adbd3c074d148430c90c2ec Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 15 Oct 2026 12:50:14 -0700 Subject: [PATCH 06/10] [DSv4.1] Remove explanatory comments from BCG image fix --- .../srt/model_executor/runner/prefill_cuda_graph_runner.py | 4 ---- python/sglang/srt/models/deepseek_v4.py | 3 --- 2 files changed, 7 deletions(-) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index e6ca100e010e..12cfc01a735e 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -2104,10 +2104,6 @@ def _execute_body_capture( full_path = self._is_full_backend def replay_layer_forward(*args, **layer_kwargs): - # The eager wrapper may remap multimodal hash placeholders to model - # token IDs. The captured body must see those IDs as well as the - # composed embeddings (e.g. for visual MoE routing and Engram masks). - # Update only the private static buffer, leaving scheduler IDs intact. prepared_ids = layer_kwargs.get("input_ids") if prepared_ids is None and args: prepared_ids = args[0] diff --git a/python/sglang/srt/models/deepseek_v4.py b/python/sglang/srt/models/deepseek_v4.py index 3aed906c37d4..8349d5a65306 100644 --- a/python/sglang/srt/models/deepseek_v4.py +++ b/python/sglang/srt/models/deepseek_v4.py @@ -4219,9 +4219,6 @@ def _forward_layers_hc_pre_from_prev( forward_batch, cp_all_tokens=cp_extend, ) - # Capture the token mask even when the dummy batch has no media: - # later graph replays can contain image tokens. Text rows retain - # the Engram result without depending on Python batch metadata. if ( self.config.model_type == "deepseek_v41" and self.config.vision_n_layers > 0 From 9a04c9f1318885f44ac9f59b51f7015d03728841 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 15 Oct 2026 15:50:13 -0700 Subject: [PATCH 07/10] [DSv4.1] Simplify prepared token ID copy --- .../srt/model_executor/runner/prefill_cuda_graph_runner.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index dbd2db55783e..98a1f85ca536 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -2118,10 +2118,7 @@ def replay_layer_forward(*args, **layer_kwargs): prepared_ids = layer_kwargs.get("input_ids") if prepared_ids is None and args: prepared_ids = args[0] - if ( - prepared_ids is not None - and prepared_ids.data_ptr() != static_forward_batch.input_ids.data_ptr() - ): + if prepared_ids is not None: static_forward_batch.input_ids[: prepared_ids.shape[0]].copy_( prepared_ids ) From 53d5912c68430940d336cbcb33bbe9318e32cbf0 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 15 Oct 2026 16:11:01 -0700 Subject: [PATCH 08/10] Limit prepared input ID synchronization to DeepSeek V4.1 --- .../srt/model_executor/runner/prefill_cuda_graph_runner.py | 5 ++++- .../unit/models/test_deepseek_v41_bcg_multimodal.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 98a1f85ca536..2e11a657fec0 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -2113,12 +2113,15 @@ def _execute_body_capture( # BCG / Full: replay the captured body, run the LM head + # logits_processor eagerly. full_path = self._is_full_backend + model_config = getattr(self.model_runner.model, "config", None) + sync_prepared_ids = getattr(model_config, "model_type", None) == "deepseek_v41" def replay_layer_forward(*args, **layer_kwargs): prepared_ids = layer_kwargs.get("input_ids") if prepared_ids is None and args: prepared_ids = args[0] - if prepared_ids is not None: + # TODO: Add general multimodal replay tests before enabling for other models. + if sync_prepared_ids and prepared_ids is not None: static_forward_batch.input_ids[: prepared_ids.shape[0]].copy_( prepared_ids ) diff --git a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py index 022d442822ba..0ec6babd6219 100644 --- a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py +++ b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py @@ -54,7 +54,10 @@ def outer_forward(ids, positions, batch): runner.model_runner = SimpleNamespace( pp_group=SimpleNamespace(is_first_rank=True), - model=SimpleNamespace(forward=outer_forward), + model=SimpleNamespace( + forward=outer_forward, + config=SimpleNamespace(model_type="deepseek_v41"), + ), ) live = SimpleNamespace(input_ids=scheduler_ids, mm_input_embeds=None) static = SimpleNamespace( From bec7847ca5b86e4f6a027b41caea202b14a5c047 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 15 Oct 2026 16:26:05 -0700 Subject: [PATCH 09/10] Initialize DeepSeek V4.1 flag in prefill graph runner --- .../srt/model_executor/runner/prefill_cuda_graph_runner.py | 7 ++++--- .../test_prefill_cuda_graph_runner_helpers.py | 3 +++ .../unit/models/test_deepseek_v41_bcg_multimodal.py | 6 ++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 2e11a657fec0..86c9f5abf696 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -332,6 +332,9 @@ def __init__(self, model_runner: ModelRunner): # --- model flags ---------------------------------------------- self.quant_config = getattr(model_runner.model, "quant_config", None) self.is_multimodal = model_runner.model_config.is_multimodal + self.is_dsv41_model = ( + model_runner.model_config.hf_config.model_type == "deepseek_v41" + ) self.enable_lora = model_runner.lora_manager is not None # Classification/reward forwards branch on return_pooled_hidden_states; # capture must use the same flag value as replay for those models. @@ -2113,15 +2116,13 @@ def _execute_body_capture( # BCG / Full: replay the captured body, run the LM head + # logits_processor eagerly. full_path = self._is_full_backend - model_config = getattr(self.model_runner.model, "config", None) - sync_prepared_ids = getattr(model_config, "model_type", None) == "deepseek_v41" def replay_layer_forward(*args, **layer_kwargs): prepared_ids = layer_kwargs.get("input_ids") if prepared_ids is None and args: prepared_ids = args[0] # TODO: Add general multimodal replay tests before enabling for other models. - if sync_prepared_ids and prepared_ids is not None: + if self.is_dsv41_model and prepared_ids is not None: static_forward_batch.input_ids[: prepared_ids.shape[0]].copy_( prepared_ids ) diff --git a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py index 7f554e313f5a..37b07ba2cb0a 100644 --- a/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py +++ b/test/registered/unit/model_executor/test_prefill_cuda_graph_runner_helpers.py @@ -386,6 +386,7 @@ def test_body_replay_uses_plural_embeds_and_skips_embedding_on_later_pp_stage(se with self.subTest(first_rank=first_rank, positional=positional): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._is_full_backend = False + runner.is_dsv41_model = False runner._qwen_bcg_hc_sidechannel = False runner._qwen_bcg_mtp_draft = False runner._input_embeds_arg_idx = 3 @@ -505,6 +506,7 @@ def _check_bcg_eager_tail(self, live_embeds): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._is_full_backend = False + runner.is_dsv41_model = False runner._qwen_bcg_hc_sidechannel = False runner._qwen_bcg_mtp_draft = True runner._input_embeds_arg_idx = None @@ -531,6 +533,7 @@ def _check_bcg_eager_tail(self, live_embeds): def test_bcg_mtp_draft_replay_fills_hc_width_input_embeds_slot(self): runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._is_full_backend = False + runner.is_dsv41_model = False runner._qwen_bcg_hc_sidechannel = False runner._qwen_bcg_mtp_draft = True runner._input_embeds_arg_idx = 3 diff --git a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py index 0ec6babd6219..9a0ab7c7d7f4 100644 --- a/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py +++ b/test/registered/unit/models/test_deepseek_v41_bcg_multimodal.py @@ -28,6 +28,7 @@ def test_replay_uses_prepared_image_ids_without_mutating_scheduler_ids(self): layer = SimpleNamespace(forward=None) runner = PrefillCudaGraphRunner.__new__(PrefillCudaGraphRunner) runner._is_full_backend = False + runner.is_dsv41_model = True runner._qwen_bcg_hc_sidechannel = False runner._use_draft_input_embeds = False runner.layer_model = layer @@ -54,10 +55,7 @@ def outer_forward(ids, positions, batch): runner.model_runner = SimpleNamespace( pp_group=SimpleNamespace(is_first_rank=True), - model=SimpleNamespace( - forward=outer_forward, - config=SimpleNamespace(model_type="deepseek_v41"), - ), + model=SimpleNamespace(forward=outer_forward), ) live = SimpleNamespace(input_ids=scheduler_ids, mm_input_embeds=None) static = SimpleNamespace( From c4eae3729852dcacff36cb1ad87f16f0d51de4d6 Mon Sep 17 00:00:00 2001 From: Oasis-Git Date: Thu, 15 Oct 2026 16:46:28 -0700 Subject: [PATCH 10/10] Simplify prepared input ID argument lookup --- .../srt/model_executor/runner/prefill_cuda_graph_runner.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py index 86c9f5abf696..119997e5bf88 100644 --- a/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py +++ b/python/sglang/srt/model_executor/runner/prefill_cuda_graph_runner.py @@ -2118,9 +2118,7 @@ def _execute_body_capture( full_path = self._is_full_backend def replay_layer_forward(*args, **layer_kwargs): - prepared_ids = layer_kwargs.get("input_ids") - if prepared_ids is None and args: - prepared_ids = args[0] + prepared_ids = layer_kwargs.get("input_ids", args[0] if args else None) # TODO: Add general multimodal replay tests before enabling for other models. if self.is_dsv41_model and prepared_ids is not None: static_forward_batch.input_ids[: prepared_ids.shape[0]].copy_(