From adf6aa19a464504f31c0ead8581f33b08decfd02 Mon Sep 17 00:00:00 2001 From: David Orman Date: Fri, 31 Jul 2026 13:31:22 -0500 Subject: [PATCH] fix: support DeepSeek-V4-Flash-0731 reasoning effort --- .../docs/references/environment_variables.mdx | 2 +- .../srt/entrypoints/openai/encoding_dsv4.py | 62 ++++++-- .../srt/entrypoints/openai/serving_chat.py | 20 ++- python/sglang/srt/environ.py | 2 + .../entrypoints/openai/test_serving_chat.py | 149 ++++++++++++++++++ 5 files changed, 217 insertions(+), 18 deletions(-) diff --git a/docs_new/docs/references/environment_variables.mdx b/docs_new/docs/references/environment_variables.mdx index 6fe524634020..63fef8cd43cb 100644 --- a/docs_new/docs/references/environment_variables.mdx +++ b/docs_new/docs/references/environment_variables.mdx @@ -712,7 +712,7 @@ SGLang supports various environment variables that can be used to configure its SGLANG_DSV4_REASONING_EFFORT - Default reasoning_effort for the DeepSeek V4 chat encoder when a request does not set it (accepts max, high; empty means unset). + Default reasoning_effort for the DeepSeek V4 chat encoder when a request does not set it. Accepts high and max; checkpoints with bundled DSpark config also accept low and use the checkpoint's low/high/max mapping. Empty means unset. "" diff --git a/python/sglang/srt/entrypoints/openai/encoding_dsv4.py b/python/sglang/srt/entrypoints/openai/encoding_dsv4.py index a19d1f4b1080..536720927de0 100644 --- a/python/sglang/srt/entrypoints/openai/encoding_dsv4.py +++ b/python/sglang/srt/entrypoints/openai/encoding_dsv4.py @@ -60,11 +60,25 @@ tool_output_template: str = "{content}" -REASONING_EFFORT_MAX = ( - "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" - "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" - "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" -) +REASONING_EFFORT_PROMPTS: Dict[str, str] = { + "low": "", + "high": ( + "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" + "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" + "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" + ), + "max": ( + "Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n" + "You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n" + "Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n" + ), +} +DEFAULT_REASONING_EFFORT = "low" +LEGACY_REASONING_EFFORT_MAP: Dict[Optional[str], str] = { + None: "low", + "high": "low", + "max": "high", +} TOOLS_TEMPLATE = """## Tools @@ -250,6 +264,7 @@ def render_message( thinking_mode: str, drop_thinking: bool = True, reasoning_effort: Optional[str] = None, + use_three_tier_reasoning_effort: bool = False, ) -> str: """ Render a single message at the given index into its encoded string form. @@ -262,7 +277,10 @@ def render_message( messages: Full list of messages in the conversation. thinking_mode: Either "chat" or "thinking". drop_thinking: Whether to drop reasoning content from earlier turns. - reasoning_effort: Optional reasoning effort level ("max", "high", or None). + reasoning_effort: In three-tier mode, one of "low", "high", "max", with + None treated as "low". Otherwise one of None, "high", "max", where + "high" adds no prefix and "max" uses the prompt now named "high". + use_three_tier_reasoning_effort: Use the newer checkpoint's three effort tiers. Returns: Encoded string for this message. @@ -290,14 +308,22 @@ def render_message( if tool_calls: tool_calls = tool_calls_from_openai_format(tool_calls) - # Reasoning effort prefix (only at index 0 in thinking mode with max effort) - assert reasoning_effort in [ - "max", - None, - "high", - ], f"Invalid reasoning effort: {reasoning_effort}" - if index == 0 and thinking_mode == "thinking" and reasoning_effort == "max": - prompt += REASONING_EFFORT_MAX + # The original checkpoint maps max to the prompt now named high; high adds + # nothing. The 0731 checkpoint defines low/high/max as distinct tiers. + if use_three_tier_reasoning_effort: + reasoning_effort = reasoning_effort or DEFAULT_REASONING_EFFORT + assert reasoning_effort in REASONING_EFFORT_PROMPTS, ( + f"Invalid reasoning effort: {reasoning_effort}, " + f"expected one of {list(REASONING_EFFORT_PROMPTS)}" + ) + else: + assert reasoning_effort in LEGACY_REASONING_EFFORT_MAP, ( + f"Invalid reasoning effort: {reasoning_effort}, " + f"expected one of {list(LEGACY_REASONING_EFFORT_MAP)}" + ) + reasoning_effort = LEGACY_REASONING_EFFORT_MAP[reasoning_effort] + if index == 0 and thinking_mode == "thinking": + prompt += REASONING_EFFORT_PROMPTS[reasoning_effort] if role == "system": prompt += system_msg_template.format(content=content or "") @@ -583,6 +609,7 @@ def encode_messages( drop_thinking: bool = True, add_default_bos_token: bool = True, reasoning_effort: Optional[str] = None, + use_three_tier_reasoning_effort: bool = False, ) -> str: """ Encode a list of messages into the DeepSeek-V4 prompt format. @@ -600,7 +627,11 @@ def encode_messages( drop_thinking: If True, drop reasoning_content from earlier assistant turns (only keep reasoning for messages after the last user message). add_default_bos_token: Whether to prepend BOS token at conversation start. - reasoning_effort: Optional reasoning effort level ("max", "high", or None). + reasoning_effort: Only takes effect in thinking mode. In three-tier mode, + one of "low", "high", "max", with None treated as "low". Otherwise + one of None, "high", "max", where "high" adds no prefix and "max" + uses the prompt now named "high". + use_three_tier_reasoning_effort: Use the newer checkpoint's three effort tiers. Returns: The encoded prompt string. @@ -640,6 +671,7 @@ def encode_messages( thinking_mode=thinking_mode, drop_thinking=effective_drop_thinking, reasoning_effort=reasoning_effort, + use_three_tier_reasoning_effort=use_three_tier_reasoning_effort, ) return prompt diff --git a/python/sglang/srt/entrypoints/openai/serving_chat.py b/python/sglang/srt/entrypoints/openai/serving_chat.py index ff09f3e0278c..c645d1cc7700 100644 --- a/python/sglang/srt/entrypoints/openai/serving_chat.py +++ b/python/sglang/srt/entrypoints/openai/serving_chat.py @@ -222,6 +222,17 @@ def __init__( # Which Python-based chat encoder (if any) bypasses apply_chat_template. # Values: "dsv32", "dsv4", or custom values set by subclass. None for default. self.chat_encoding_spec = self._resolve_chat_encoding_spec() + self._dsv4_uses_three_tier_reasoning_effort = False + if self.chat_encoding_spec == "dsv4": + from sglang.srt.speculative.dspark_components.dspark_config import ( + checkpoint_bundles_dspark_draft, + ) + + self._dsv4_uses_three_tier_reasoning_effort = ( + checkpoint_bundles_dspark_draft( + getattr(self.tokenizer_manager.model_config, "hf_config", None) + ) + ) # Resolve the env-configured Inkling effort default once: the env var is # frozen for the server's lifetime, and a misconfigured value should @@ -986,7 +997,6 @@ def _apply_jinja_template( # Default encoding (dsv4/dsv32) if self.chat_encoding_spec == "dsv4": - # V4 encoder only accepts "max" / "high" / None. # OpenAI protocol defaults to "medium" which V4 rejects; drop it. # Fallback: if request didn't set it, try env SGLANG_DSV4_REASONING_EFFORT. effort_source = request.reasoning_effort @@ -994,8 +1004,13 @@ def _apply_jinja_template( env_val = envs.SGLANG_DSV4_REASONING_EFFORT.get() if env_val: effort_source = env_val + valid_efforts = ( + ("low", "high", "max") + if self._dsv4_uses_three_tier_reasoning_effort + else ("high", "max") + ) v4_reasoning_effort = ( - effort_source if effort_source in ("max", "high") else None + effort_source if effort_source in valid_efforts else None ) if request.task is not None: encoding_dsv4.attach_task_to_last_user_message( @@ -1005,6 +1020,7 @@ def _apply_jinja_template( messages, thinking_mode=thinking_mode, reasoning_effort=v4_reasoning_effort, + use_three_tier_reasoning_effort=self._dsv4_uses_three_tier_reasoning_effort, ) prompt_ids = self.tokenizer_manager.tokenizer.encode(real_input) else: diff --git a/python/sglang/srt/environ.py b/python/sglang/srt/environ.py index b46a8063ef3f..24f73796984a 100644 --- a/python/sglang/srt/environ.py +++ b/python/sglang/srt/environ.py @@ -1086,6 +1086,8 @@ class Envs: SGLANG_DSV4_FP4_DEQUANT = EnvBool(False) # Default reasoning_effort for dsv4 chat encoder when request doesn't set it. # Accepts "", "max", "high" (empty string means unset); other values filtered to None. + # "low" is additionally accepted only for checkpoints selected by the bundled + # DSpark marker, which use the three-tier low/high/max mapping. SGLANG_DSV4_REASONING_EFFORT = EnvStr("") # Quantize the SWA fp8 KV cache from bf16-rounded values (matches # trainer-side QAT and the DSA-CP path) instead of fp32 registers. diff --git a/test/registered/unit/entrypoints/openai/test_serving_chat.py b/test/registered/unit/entrypoints/openai/test_serving_chat.py index 236cf394887c..e5052de35bf3 100644 --- a/test/registered/unit/entrypoints/openai/test_serving_chat.py +++ b/test/registered/unit/entrypoints/openai/test_serving_chat.py @@ -50,6 +50,10 @@ def __init__(self): # Mock hf_config for _resolve_chat_encoding_spec check mock_hf_config = Mock() mock_hf_config.architectures = ["LlamaForCausalLM"] + mock_hf_config.dspark_block_size = None + mock_hf_config.dspark_markov_rank = None + mock_hf_config.dspark_noise_token_id = None + mock_hf_config.dspark_target_layer_ids = None self.model_config.hf_config = mock_hf_config self.chat_template_name: Optional[str] = "llama-3" @@ -1256,6 +1260,68 @@ def test_dpsk_v32_encoding_path(self): serving_chat = OpenAIServingChat(tm, TemplateManager()) self.assertEqual(serving_chat.chat_encoding_spec, "dsv4") + def test_dsv4_0731_reasoning_effort_detection(self): + from sglang.srt.parser.template_manager import TemplateManager + + tm = _MockTokenizerManager() + mock_hf_config = tm.model_config.hf_config + mock_hf_config.architectures = ["DeepseekV4ForCausalLM"] + + serving_chat = OpenAIServingChat(tm, TemplateManager()) + self.assertFalse(serving_chat._dsv4_uses_three_tier_reasoning_effort) + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hello"}], + reasoning_effort="max", + ) + serving_chat._process_messages(request, is_multimodal=False) + legacy_prompt = tm.tokenizer.encode.call_args.args[0] + self.assertIn("Reasoning Effort: Absolute maximum", legacy_prompt) + self.assertNotIn("Reasoning Effort: Beyond maximum", legacy_prompt) + for effort in ("low", "medium"): + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hello"}], + reasoning_effort=effort, + ) + serving_chat._process_messages(request, is_multimodal=False) + self.assertNotIn("Reasoning Effort:", tm.tokenizer.encode.call_args.args[0]) + + mock_hf_config.dspark_block_size = 5 + serving_chat = OpenAIServingChat(tm, TemplateManager()) + self.assertTrue(serving_chat._dsv4_uses_three_tier_reasoning_effort) + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hello"}], + reasoning_effort="max", + ) + serving_chat._process_messages(request, is_multimodal=False) + prompt_0731 = tm.tokenizer.encode.call_args.args[0] + self.assertIn("Reasoning Effort: Beyond maximum", prompt_0731) + + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hello"}], + reasoning_effort="medium", + ) + serving_chat._process_messages(request, is_multimodal=False) + self.assertNotIn("Reasoning Effort:", tm.tokenizer.encode.call_args.args[0]) + + mock_hf_config.dspark_block_size = None + mock_hf_config.dspark_markov_rank = 2 + serving_chat = OpenAIServingChat(tm, TemplateManager()) + self.assertTrue(serving_chat._dsv4_uses_three_tier_reasoning_effort) + request = ChatCompletionRequest( + model="x", + messages=[{"role": "user", "content": "Hello"}], + reasoning_effort="max", + ) + serving_chat._process_messages(request, is_multimodal=False) + self.assertIn( + "Reasoning Effort: Beyond maximum", tm.tokenizer.encode.call_args.args[0] + ) + mock_hf_config.dspark_markov_rank = None + # ------------- dsv4 task + latest_reminder ------------- def test_dsv4_task_field_schema(self): """Top-level `task` accepts the 6 DS task tokens and rejects others.""" @@ -1406,6 +1472,89 @@ def test_dsv4_task_and_reminder_encode_end_to_end(self): ) self.assertIn("<｜Assistant｜>", out) + def test_dsv4_reasoning_effort_checkpoint_compatibility(self): + from sglang.srt.entrypoints.openai import encoding_dsv4 + + messages = [{"role": "user", "content": "Hello"}] + base = "<｜begin▁of▁sentence｜><｜User｜>Hello" "<｜Assistant｜>" + high_prefix = ( + "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" + "You MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\n" + "Explicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n" + ) + max_prefix = ( + "Reasoning Effort: Beyond maximum — exhaustive, relentless, and uncompromising.\n" + "You MUST reason with the utmost depth and rigor, leaving absolutely nothing to chance: exhaustively decompose the problem into its most fundamental components, trace every causal chain to its root, and resolve the underlying cause rather than any surface symptom.\n" + "Do not stop reasoning until you have independently verified the solution from multiple angles and are certain that no assumption remains unchecked and no error remains undiscovered.\n\n" + ) + + self.assertEqual( + encoding_dsv4.encode_messages(messages, thinking_mode="thinking"), + base, + ) + self.assertEqual( + encoding_dsv4.encode_messages( + messages, thinking_mode="thinking", reasoning_effort="high" + ), + base, + ) + self.assertEqual( + encoding_dsv4.encode_messages( + messages, thinking_mode="thinking", reasoning_effort="max" + ), + f"<｜begin▁of▁sentence｜>{high_prefix}{base[len('<｜begin▁of▁sentence｜>'):]}", + ) + + with self.assertRaises(AssertionError): + encoding_dsv4.encode_messages( + messages, thinking_mode="thinking", reasoning_effort="low" + ) + + self.assertEqual( + encoding_dsv4.encode_messages( + messages, + thinking_mode="thinking", + use_three_tier_reasoning_effort=True, + ), + base, + ) + self.assertEqual( + encoding_dsv4.encode_messages( + messages, + thinking_mode="thinking", + reasoning_effort="low", + use_three_tier_reasoning_effort=True, + ), + base, + ) + self.assertEqual( + encoding_dsv4.encode_messages( + messages, + thinking_mode="thinking", + reasoning_effort="high", + use_three_tier_reasoning_effort=True, + ), + f"<｜begin▁of▁sentence｜>{high_prefix}{base[len('<｜begin▁of▁sentence｜>'):]}", + ) + self.assertEqual( + encoding_dsv4.encode_messages( + messages, + thinking_mode="thinking", + reasoning_effort="max", + use_three_tier_reasoning_effort=True, + ), + f"<｜begin▁of▁sentence｜>{max_prefix}{base[len('<｜begin▁of▁sentence｜>'):]}", + ) + self.assertEqual( + encoding_dsv4.encode_messages( + messages, + thinking_mode="chat", + reasoning_effort="max", + use_three_tier_reasoning_effort=True, + ), + "<｜begin▁of▁sentence｜><｜User｜>Hello<｜Assistant｜>", + ) + def test_streaming_abort_yields_error(self): """Test that an abort finish reason during streaming correctly yields an error and stops.""" err_msg = "Aborted by scheduler"