From 0e4ea01da8fa8df00d187d0255da2352a9c4deaf Mon Sep 17 00:00:00 2001 From: Aarne Talman Date: Thu, 15 Oct 2026 18:11:35 +0000 Subject: [PATCH] [Fix] deterministic: Disable mixed chunked prefill in deterministic mode With --enable-mixed-chunk, a batch can hold prefill chunks and decode tokens together, and the whole batch runs the prefill (extend) attention path. A decode token's result then depends on whether a prefill happened to share its step, which deterministic inference must not allow: the extend and decode kernels reduce in different orders (the prefill vs decode difference that test_deterministic's p_vs_d mode measures), and with per-phase backends they can be different backends. Deterministic mode now switches --enable-mixed-chunk off with a warning, as it already does for --enable-aiter-allreduce-fusion. --- python/sglang/srt/arg_groups/attention_hook.py | 12 ++++++++++++ test/registered/unit/test_model_overrides.py | 17 +++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/python/sglang/srt/arg_groups/attention_hook.py b/python/sglang/srt/arg_groups/attention_hook.py index 92a23018a057..025a0eaf9fcd 100644 --- a/python/sglang/srt/arg_groups/attention_hook.py +++ b/python/sglang/srt/arg_groups/attention_hook.py @@ -606,6 +606,18 @@ def handle_deterministic_inference(server_args: Any): enable_aiter_allreduce_fusion=False, ) + # A mixed batch runs its decode tokens through the prefill (extend) kernels, + # so a decode step's result would depend on whether a prefill shares the batch. + if cfg.enable_mixed_chunk: + logger.warning( + "Disable --enable-mixed-chunk because deterministic inference is enabled." + ) + declare_resolution( + server_args, + "_handle_deterministic_inference", + enable_mixed_chunk=False, + ) + # Moved to the resolution pipeline (arg_groups/overrides.py: # _deterministic_allreduce_fusion_disable), invoked here at its # legacy slot. diff --git a/test/registered/unit/test_model_overrides.py b/test/registered/unit/test_model_overrides.py index 2f960749a394..dfe0a48193c1 100644 --- a/test/registered/unit/test_model_overrides.py +++ b/test/registered/unit/test_model_overrides.py @@ -1799,6 +1799,23 @@ def test_deterministic_inference_forces_pytorch_sampling(self): ) self.assertEqual(self._leaf("attention_backend"), deterministic_fills[0]) + def test_deterministic_inference_disables_mixed_chunk(self): + """A mixed batch runs decode tokens through the extend kernels, so deterministic + mode must not put prefill and decode tokens in one batch.""" + sa = self._construct( + "LlamaForCausalLM", + "llama", + enable_deterministic_inference=True, + enable_mixed_chunk=True, + ) + self.assertFalse(resolution_result(sa, "enable_mixed_chunk")) + self.assertTrue( + resolution_result( + self._construct("LlamaForCausalLM", "llama", enable_mixed_chunk=True), + "enable_mixed_chunk", + ) + ) + def test_deterministic_incompatible_backend_raises(self): from sglang.srt.arg_groups.overrides import ( ResolvedView,