diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a641bacdd84..d1b42920a4e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,10 @@ default_language_version: python: python3 +# The vendored upstream module stays byte-diffable against its source; keep +# formatters and linters away so upstream-tracking diffs remain minimal. +exclude: '^miles/utils/chat_template_utils/templates/encoding_dsv32\.py$' + ci: autofix_prs: true autoupdate_commit_msg: '[pre-commit.ci] pre-commit suggestions' diff --git a/miles/backends/sglang_utils/arguments.py b/miles/backends/sglang_utils/arguments.py index 02939d4f63a..56fd8fd2b71 100644 --- a/miles/backends/sglang_utils/arguments.py +++ b/miles/backends/sglang_utils/arguments.py @@ -107,9 +107,15 @@ def new_add_argument_wrapper(*name_or_flags, **kwargs): # Avoid double prefixing if dest somehow already starts with sglang_ if not original_dest.startswith("sglang_"): final_kwargs["dest"] = f"sglang_{original_dest}" - # If 'dest' is not explicitly provided (or is None/not a string), - # argparse will derive 'dest' from the (now prefixed) flag names. - # E.g., if the first flag is "--sglang-foo-bar", argparse sets dest to "sglang_foo_bar". + elif "dest" not in final_kwargs: + # argparse derives dest from the first alias, so store parallel sizes under SGLang's short field names. + for item_flag in name_or_flags: + if not isinstance(item_flag, str) or not item_flag.startswith("--"): + continue + canonical_dest = item_flag[2:].replace("-", "_") + if canonical_dest in ("tp_size", "dp_size", "pp_size", "ep_size"): + final_kwargs["dest"] = f"sglang_{canonical_dest}" + break old_add_argument(*new_name_or_flags_list, **final_kwargs) diff --git a/miles/utils/chat_template_utils/deepseek.py b/miles/utils/chat_template_utils/deepseek.py index a985a001862..58a4d8c68b3 100644 --- a/miles/utils/chat_template_utils/deepseek.py +++ b/miles/utils/chat_template_utils/deepseek.py @@ -1,11 +1,11 @@ """Shared bridge for the DeepSeek official-encoder families (V3.2, V4). -Neither family ships a jinja chat_template: sglang renders their prompts -through per-family ``encoding_dsv*`` modules that share one calling -convention, and miles' ``apply_chat_template`` routes any matching tokenizer -here so training-side renders stay byte-aligned with what the runtime -serves. Each family is one ``DeepSeekFamily`` instance wrapping its encoder -module; everything else is shared. +Neither family ships a jinja chat_template: V4 renders through sglang's +``encoding_dsv4``, while V3.2 uses miles' vendored +``templates.encoding_dsv32``. Both modules share one calling convention, +and miles' ``apply_chat_template`` routes any matching tokenizer here. Each +family is one ``DeepSeekFamily`` instance wrapping its encoder module; +everything else is shared. """ from __future__ import annotations @@ -16,9 +16,11 @@ import os from typing import Any -from sglang.srt.entrypoints.openai import encoding_dsv4, encoding_dsv32 +from sglang.srt.entrypoints.openai import encoding_dsv4 from sglang.srt.entrypoints.openai.protocol import Tool +from miles.utils.chat_template_utils.templates import encoding_dsv32 + _ASSISTANT_SP_TOKEN = "<|Assistant|>" diff --git a/miles/utils/chat_template_utils/templates/__init__.py b/miles/utils/chat_template_utils/templates/__init__.py new file mode 100644 index 00000000000..222cd793ba5 --- /dev/null +++ b/miles/utils/chat_template_utils/templates/__init__.py @@ -0,0 +1 @@ +"""Bundled fixed chat templates and the vendored official DeepSeek V3.2 encoder.""" diff --git a/miles/utils/chat_template_utils/templates/encoding_dsv32.py b/miles/utils/chat_template_utils/templates/encoding_dsv32.py new file mode 100644 index 00000000000..500f33bb7f5 --- /dev/null +++ b/miles/utils/chat_template_utils/templates/encoding_dsv32.py @@ -0,0 +1,489 @@ +# Vendored from sglang 0.5.14.dev37+gf8cfad3 srt/entrypoints/openai/encoding_dsv32.py so miles can modify the render behavior locally. +# Adapted from https://huggingface.co/deepseek-ai/DeepSeek-V3.2/blob/main/encoding/encoding_dsv32.py +import copy +import json +import re +from typing import Any, Dict, List, Optional, Tuple, Union + + +class DS32EncodingError(Exception): + pass + + +TOOLS_SYSTEM_TEMPLATE = """## Tools +You have access to a set of tools you can use to answer the user's question. +You can invoke functions by writing a "<{dsml_token}function_calls>" block like the following as part of your reply to the user: +<{dsml_token}function_calls> +<{dsml_token}invoke name="$FUNCTION_NAME"> +<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml_token}invoke name="$FUNCTION_NAME2"> +... + + +String and scalar parameters should be specified as is without any escaping or quotes, while lists and objects should use JSON format. The "string" attribute should be set to "true" for string type parameters and "false" for other types (numbers, booleans, arrays, objects). +If the thinking_mode is enabled, then after function results you should strongly consider outputting a thinking block. Here is an example: +<{dsml_token}function_calls> +... + + +... + +{thinking_start_token}...thinking about results{thinking_end_token} +Here are the functions available in JSONSchema format: + +{tool_schemas} + +""" + +bos_token: str = "<|begin▁of▁sentence|>" +eos_token: str = "<|end▁of▁sentence|>" +thinking_start_token: str = "" +thinking_end_token: str = "" +dsml_token: str = "|DSML|" +system_msg_template: str = "{content}" +user_msg_template: str = "<|User|>{content}<|Assistant|>" +assistant_msg_template: str = "{reasoning}{content}{tool_calls}<|end▁of▁sentence|>" +thinking_template = "{reasoning_content}" + +response_format_template: str = ( + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}" +) +tool_call_template: str = ( + '<{dsml_token}invoke name="{name}">\n{arguments}\n' +) +tool_calls_template = ( + "<{dsml_token}function_calls>\n{tool_calls}\n" +) + +tool_output_template: str = "\n{content}" + + +def to_json(value: Any) -> str: + try: + return json.dumps(value, ensure_ascii=False) + except: + return json.dumps(value, ensure_ascii=True) + + +def tools_from_openai_format(tools): + return [tool["function"] for tool in tools] + + +def tool_calls_from_openai_format(tool_calls): + return [ + { + "name": tool_call["function"]["name"], + "arguments": tool_call["function"]["arguments"], + } + for tool_call in tool_calls + ] + + +def tool_calls_to_openai_format(tool_calls): + return [ + { + "type": "function", + "function": { + "name": tool_call["name"], + "arguments": tool_call["arguments"], + }, + } + for tool_call in tool_calls + ] + + +def encode_arguments_to_dsml(tool_call: Dict[str, str]) -> str: + p_dsml_template = """<{dsml_token}parameter name="{key}" string="{is_str}">{value}""" + P_dsml_strs = [] + + raw_arguments = tool_call["arguments"] + arguments = ( + json.loads(raw_arguments) if isinstance(raw_arguments, str) else raw_arguments + ) + if not isinstance(arguments, dict): + raise ValueError( + "Assistant tool call function.arguments must be a JSON object." + ) + + for k, v in arguments.items(): + p_dsml_str = p_dsml_template.format( + dsml_token=dsml_token, + key=k, + is_str="true" if isinstance(v, str) else "false", + value=v if isinstance(v, str) else to_json(v), + ) + + P_dsml_strs.append(p_dsml_str) + + return "\n".join(P_dsml_strs) + + +def decode_dsml_to_arguments( + tool_name: str, tool_args: Dict[str, Tuple[str, str]] +) -> Dict[str, str]: + def _decode_value(key: str, value: str, string: str): + if string == "true": + value = to_json(value) + return f"{to_json(key)}: {value}" + + tool_args_json = ( + "{" + + ", ".join( + [_decode_value(k, v, string=is_str) for k, (v, is_str) in tool_args.items()] + ) + + "}" + ) + return dict(name=tool_name, arguments=tool_args_json) + + +def render_tools(tools: List[Dict[str, Union[str, Dict[str, Any]]]]) -> str: + tools_json = [to_json(t) for t in tools] + + return TOOLS_SYSTEM_TEMPLATE.format( + tool_schemas="\n".join(tools_json), + dsml_token=dsml_token, + thinking_start_token=thinking_start_token, + thinking_end_token=thinking_end_token, + ) + + +def find_last_user_index(messages: List[Dict[str, Any]]) -> int: + last_user_index = -1 + for idx in range(len(messages) - 1, -1, -1): + if messages[idx].get("role") in ["user", "developer"]: + last_user_index = idx + break + return last_user_index + + +def render_message( + index: int, messages: List[Dict[str, Any]], thinking_mode: str, drop_thinking: bool = True +) -> str: + if not (0 <= index < len(messages)): + raise DS32EncodingError( + f"Index {index} out of range for messages list of length {len(messages)}" + ) + if thinking_mode not in ["chat", "thinking"]: + raise DS32EncodingError(f"Invalid thinking_mode `{thinking_mode}`") + + prompt = "" + msg = messages[index] + last_user_idx = find_last_user_index(messages) + + role = msg.get("role") + content = msg.get("content") + tools = msg.get("tools") + response_format = msg.get("response_format") + tool_calls = msg.get("tool_calls") + reasoning_content = msg.get("reasoning_content") + + if tools: + tools = tools_from_openai_format(tools) + if tool_calls: + tool_calls = tool_calls_from_openai_format(tool_calls) + + if role == "system": + prompt += system_msg_template.format(content=content or "") + if tools: + prompt += "\n\n" + render_tools(tools) + + if response_format: + prompt += "\n\n" + response_format_template.format( + schema=to_json(response_format) + ) + + elif role == "developer": + if not content: + raise DS32EncodingError(f"Invalid message for role `{role}`: {msg}") + content_developer = "" + if tools: + content_developer += "\n\n" + render_tools(tools) + + if response_format: + content_developer += "\n\n" + response_format_template.format( + schema=to_json(response_format) + ) + + content_developer += "\n\n# The user's message is: {}".format(content) + + prompt += user_msg_template.format(content=content_developer) + # Miles modification: with drop_thinking=False every turn keeps its + # thinking block, so the assistant opener is regardless of + # position (mirrors encoding_dsv4's generation-suffix gate). + if not drop_thinking and thinking_mode == "thinking": + prompt += thinking_start_token + elif drop_thinking and index == last_user_idx and thinking_mode == "thinking": + prompt += thinking_start_token + else: + prompt += thinking_end_token + + elif role == "user": + prompt += user_msg_template.format(content=content) + + if not drop_thinking and thinking_mode == "thinking": + prompt += thinking_start_token + elif drop_thinking and index == last_user_idx and thinking_mode == "thinking": + prompt += thinking_start_token + else: + prompt += thinking_end_token + + elif role == "tool": + prev_assistant_idx = index - 1 + assistant_msg = messages[prev_assistant_idx] + while prev_assistant_idx >= 0 and assistant_msg.get("role") == "tool": + prev_assistant_idx -= 1 + assistant_msg = messages[prev_assistant_idx] + + if not ( + index == 0 + or (prev_assistant_idx >= 0 and assistant_msg.get("role") == "assistant") + ): + raise DS32EncodingError(f"Invalid messages at {index}:\n{assistant_msg}") + + tool_call_order = index - prev_assistant_idx + assistant_tool_calls = assistant_msg.get("tool_calls") + if not (assistant_tool_calls and len(assistant_tool_calls) >= tool_call_order): + raise DS32EncodingError("No tool calls but found tool output") + + if tool_call_order == 1: + prompt += "\n\n" + + prompt += tool_output_template.format(content=content) + + if tool_call_order == len(assistant_tool_calls): + prompt += "\n" + + if not drop_thinking and thinking_mode == "thinking": + prompt += "\n\n" + thinking_start_token + elif drop_thinking and index >= last_user_idx and thinking_mode == "thinking": + prompt += "\n\n" + thinking_start_token + else: + prompt += "\n\n" + thinking_end_token + + elif role == "assistant": + prev_assistant_idx = index + thinking_part = "" + + tool_calls_content = "" + if tool_calls: + tool_calls = [ + tool_call_template.format( + dsml_token=dsml_token, + name=tool_call.get("name"), + arguments=encode_arguments_to_dsml(tool_call), + ) + for tool_call in tool_calls + ] + tool_calls_content += "\n\n" + tool_calls_template.format( + dsml_token=dsml_token, tool_calls="\n".join(tool_calls) + ) + + summary_content = content or "" + + if thinking_mode == "thinking" and index > last_user_idx: + if not (reasoning_content or tool_calls): + raise DS32EncodingError( + f"ThinkingMode: {thinking_mode}, invalid message without reasoning_content/tool_calls `{msg}` after last user message" + ) + # Miles modification: drop_thinking=False keeps every assistant's + # thinking block in the render (upstream only renders it after the + # last user turn), which makes the render append-only across user + # appends; mirrors encoding_dsv4's assistant gate. + if thinking_mode == "thinking" and (not drop_thinking or index > last_user_idx): + thinking_part = ( + thinking_template.format(reasoning_content=reasoning_content or "") + + thinking_end_token + ) + + prompt += assistant_msg_template.format( + reasoning=thinking_part, + content=summary_content, + tool_calls=tool_calls_content, + ) + else: + raise NotImplementedError(f"Unknown role: {role}") + + return prompt + + +def drop_thinking_messages( + messages: List[Dict[str, Any]], last_user_idx: Optional[int] = None +) -> List[Dict[str, Any]]: + messages_wo_thinking: List[Dict[str, Any]] = [] + last_user_idx = ( + find_last_user_index(messages) if last_user_idx is None else last_user_idx + ) + for idx, msg in enumerate(messages): + role = msg.get("role") + if role in ["user", "system", "tool"] or idx >= last_user_idx: + messages_wo_thinking.append(msg) + continue + + elif role == "assistant": + msg_wo_thinking = copy.copy(msg) + msg_wo_thinking.pop("reasoning_content", None) + messages_wo_thinking.append(msg_wo_thinking) + + return messages_wo_thinking + + +def encode_messages( + messages: List[Dict[str, Any]], + thinking_mode: str, + context: Optional[List[Dict[str, Any]]] = None, + drop_thinking: bool = True, + add_default_bos_token: bool = True, +) -> str: + context = context if context else [] + full_messages = context + messages + + prompt = bos_token if add_default_bos_token and len(context) == 0 else "" + + if thinking_mode == "thinking" and drop_thinking: + full_messages = drop_thinking_messages(full_messages) + + for idx in range(len(messages)): + prompt += render_message( + idx + len(context), + full_messages, + thinking_mode=thinking_mode, + drop_thinking=drop_thinking, + ) + + return prompt + + +def _read_until_stop( + index: int, text: str, stop: List[str] +) -> Tuple[int, str, Optional[str]]: + min_pos = len(text) + matched_stop = None + + for s in stop: + pos = text.find(s, index) + if pos != -1 and pos < min_pos: + min_pos = pos + matched_stop = s + + if matched_stop: + content = text[index:min_pos] + return min_pos + len(matched_stop), content, matched_stop + else: + content = text[index:] + return len(text), content, None + + +def parse_tool_calls(index: int, text: str): + tool_calls: List[Dict[str, Any]] = [] + stop_token = None + tool_calls_end_token = f"" + + while index < len(text): + index, _, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}invoke", tool_calls_end_token] + ) + if _ != ">\n": + raise DS32EncodingError("Tool call format error") + + if stop_token == tool_calls_end_token: + break + + if stop_token is None: + raise DS32EncodingError("Missing special token") + + index, tool_name_content, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}parameter", f"\n$', tool_name_content, flags=re.DOTALL + ) + if len(p_tool_name) != 1: + raise DS32EncodingError("Tool name format error") + tool_name = p_tool_name[0] + + tool_args: Dict[str, Tuple[str, str]] = {} + while stop_token == f"<{dsml_token}parameter": + index, param_content, stop_token = _read_until_stop( + index, text, [f"/{dsml_token}parameter"] + ) + + param_kv = re.findall( + r'^ name="(.*?)" string="(true|false)">(.*?)<$', + param_content, + flags=re.DOTALL, + ) + if len(param_kv) != 1: + raise DS32EncodingError("Parameter format error") + param_name, string, param_value = param_kv[0] + + if param_name in tool_args: + raise DS32EncodingError("Duplicate parameter name") + tool_args[param_name] = (param_value, string) + + index, content, stop_token = _read_until_stop( + index, text, [f"<{dsml_token}parameter", f"\n": + raise DS32EncodingError("Parameter format error") + + tool_call = decode_dsml_to_arguments(tool_name=tool_name, tool_args=tool_args) + tool_calls.append(tool_call) + + return index, stop_token, tool_calls + + +# NOTE: This function is designed to parse only correctly formatted string and will not attempt to correct malformed output that may be generated by the model. +def parse_message_from_completion_text(text: str, thinking_mode: str): + summary_content, reasoning_content, tool_calls = "", "", [] + index, stop_token = 0, None + tool_calls_start_token = f"\n\n<{dsml_token}function_calls" + + is_thinking, is_tool_calling = thinking_mode == "thinking", False + + if is_thinking: + index, content_delta, stop_token = _read_until_stop( + index, text, [thinking_end_token, tool_calls_start_token] + ) + reasoning_content = content_delta + if stop_token != thinking_end_token: + raise DS32EncodingError("Invalid thinking format") + + index, content_delta, stop_token = _read_until_stop( + index, text, [eos_token, tool_calls_start_token] + ) + summary_content = content_delta + if stop_token == tool_calls_start_token: + is_tool_calling = True + else: + if stop_token != eos_token: + raise DS32EncodingError("Invalid summary format") + + if is_tool_calling: + index, stop_token, tool_calls = parse_tool_calls(index, text) + + index, tool_ends_text, stop_token = _read_until_stop(index, text, [eos_token]) + if tool_ends_text: + raise DS32EncodingError("Unexpected content after tool calls") + + if not (len(text) == index and stop_token in [eos_token, None]): + raise DS32EncodingError("Unexpected content at end") + + for sp_token in [ + bos_token, + eos_token, + thinking_start_token, + thinking_end_token, + dsml_token, + ]: + if sp_token in summary_content or sp_token in reasoning_content: + raise DS32EncodingError("Unexpected special token in content") + + return { + "role": "assistant", + "content": summary_content, + "reasoning_content": reasoning_content, + "tool_calls": tool_calls_to_openai_format(tool_calls), + } diff --git a/miles/utils/chat_template_utils/tito_tokenizer.py b/miles/utils/chat_template_utils/tito_tokenizer.py index 7e044ab49f3..75d4c0d704f 100644 --- a/miles/utils/chat_template_utils/tito_tokenizer.py +++ b/miles/utils/chat_template_utils/tito_tokenizer.py @@ -641,20 +641,20 @@ class MinimaxM27TITOTokenizer(MinimaxM25TITOTokenizer): class DeepSeekV32TITOTokenizer(TITOTokenizer): - """DeepSeek V3.2 — official encoder via sglang's ``encoding_dsv32``. - - V3.2 ships no jinja chat_template; sglang renders prompts through - ``encoding_dsv32.encode_messages``, and miles' ``apply_chat_template`` routes - any V3.2 tokenizer to the thin ``chat_template_utils.deepseek`` bridge. - TITO incremental tokenization rides that same bridge so it stays - byte-aligned with what the runtime serves. - - Only the ``{tool}`` surface is registered. DeepSeek's official - ``encoding_dsv32`` gates an assistant's thinking block on - ``index > last_user_idx``: appending a *user* turn re-classifies every prior - assistant as "before last user" and strips its thinking block, which is not - append-only. Tool-only append is safe because ``find_last_user_index`` - ignores tool roles, so the last-user position never moves. + """DeepSeek V3.2 — miles' vendored copy of the official ``encoding_dsv32``. + + V3.2 ships no jinja chat_template; prompts render through + ``templates.encoding_dsv32.encode_messages``, and miles' + ``apply_chat_template`` routes any V3.2 tokenizer to the thin + ``chat_template_utils.deepseek`` bridge. TITO incremental tokenization + rides that same bridge. + + Upstream ``encoding_dsv32`` gates every thinking block on + ``last_user_idx``: appending a *user* turn re-classifies every prior + assistant as "before last user" and strips its thinking block, which is + not append-only. The vendored copy honors ``drop_thinking=False`` at the + render level (like ``encoding_dsv4``), so every surface pins it and the + ``{tool, user}`` surface becomes legal. """ reasoning_parser = "deepseek-v3" @@ -664,6 +664,12 @@ class DeepSeekV32TITOTokenizer(TITOTokenizer): FixedTemplateRow( allowed_roles=frozenset({"tool"}), template=None, + extra_kwargs={"drop_thinking": False}, + ), + FixedTemplateRow( + allowed_roles=frozenset({"tool", "user"}), + template=None, + extra_kwargs={"drop_thinking": False}, ), ) @@ -689,6 +695,10 @@ def __init__( }, allowed_append_roles=allowed_append_roles, ) + self.chat_template_kwargs = { + **self.chat_template_kwargs, + "thinking": deepseek.V32.render_thinking_enabled(self.chat_template_kwargs), + } # --------------------------------------------------------------------------- diff --git a/miles/utils/test_utils/session_verify_runner.py b/miles/utils/test_utils/session_verify_runner.py index 0ec4a706720..23468157f41 100644 --- a/miles/utils/test_utils/session_verify_runner.py +++ b/miles/utils/test_utils/session_verify_runner.py @@ -70,7 +70,8 @@ "ci_test": True, "colocate": True, "train_backend": "fsdp", - "sglang_expert_parallel_size": 1, + "sglang_ep_size": 1, + "enable_spec": False, } @@ -166,8 +167,17 @@ def namespace_to_train_args(ns: argparse.Namespace) -> str: # DeepSeek V3.2 (and other NSA/MoE archs) requires expert-parallel > 1 in # sglang; the default is 1, which is fatal at engine init. Only emit the # flag when the caller asks for ep>1 so single-expert models stay untouched. - if ns.sglang_expert_parallel_size > 1: - parts.append(f"--sglang-expert-parallel-size {ns.sglang_expert_parallel_size}") + if ns.sglang_ep_size > 1: + parts.append(f"--sglang-expert-parallel-size {ns.sglang_ep_size}") + if ns.enable_spec: + parts.extend( + [ + "--sglang-speculative-algorithm EAGLE", + "--sglang-speculative-num-steps 2", + "--sglang-speculative-eagle-topk 1", + "--sglang-speculative-num-draft-tokens 3", + ] + ) if ns.use_session_server: parts.append("--use-session-server") if ns.debug_rollout_only: diff --git a/tests/e2e/sglang/test_session_server_multi_role/_common.py b/tests/e2e/sglang/test_session_server_multi_role/_common.py index 772c6e8b4c1..25f5b99fb77 100644 --- a/tests/e2e/sglang/test_session_server_multi_role/_common.py +++ b/tests/e2e/sglang/test_session_server_multi_role/_common.py @@ -3,7 +3,7 @@ Each test file in this directory owns a single ``ModelConfig`` and drives it through ``run_one(cfg)``. The runner is a thin wrapper around ``miles.utils.test_utils.session_verify_runner.run_session_verify`` with the -4-GPU H200 ``num_gpus`` override applied centrally. +model-specific GPU topology applied centrally. """ import argparse @@ -28,6 +28,7 @@ class ModelConfig: # sglang expert-parallel size. MoE archs like DeepSeek V4 hit a fused-moe # shape assert at ep=1; mirror the family's serving recipe (usually =tp). ep_size: int = 1 + enable_spec: bool = False cycles: int = 3 n_samples_per_prompt: int = 4 # Soft-threshold override for assistant_text mismatch ratio. Default @@ -44,7 +45,8 @@ class ModelConfig: def run_one(cfg: ModelConfig) -> None: invariants = dict(SESSION_VERIFY_INVARIANT_ARGS) - invariants["sglang_expert_parallel_size"] = cfg.ep_size + invariants["sglang_ep_size"] = cfg.ep_size + invariants["enable_spec"] = cfg.enable_spec args = argparse.Namespace( hf_checkpoint=cfg.model_name, tito_model=cfg.tito_model, diff --git a/tests/e2e/sglang/test_session_server_multi_role/test_deepseekv4.py b/tests/e2e/sglang/test_session_server_multi_role/test_deepseekv4.py index 48f154306c6..6a7516c4ba1 100644 --- a/tests/e2e/sglang/test_session_server_multi_role/test_deepseekv4.py +++ b/tests/e2e/sglang/test_session_server_multi_role/test_deepseekv4.py @@ -16,6 +16,7 @@ tp_size=4, # V4-Flash serving recipe (scripts/run_deepseek_v4.py): tp=4, ep=4. ep_size=4, + enable_spec=True, cycles=2, # V4 sorts tool_result blocks by the preceding assistant's tool_calls # order, so a sentinel tool_call_id would not roundtrip; use the diff --git a/tests/fast/utils/chat_template_utils/test_deepseek_v32.py b/tests/fast/utils/chat_template_utils/test_deepseek_v32.py index 20f744b9ca3..ca3ed2298ac 100644 --- a/tests/fast/utils/chat_template_utils/test_deepseek_v32.py +++ b/tests/fast/utils/chat_template_utils/test_deepseek_v32.py @@ -17,8 +17,10 @@ import json import pytest +from sglang.srt.entrypoints.openai import encoding_dsv32 as upstream from miles.utils.chat_template_utils import apply_chat_template, deepseek +from miles.utils.chat_template_utils.templates import encoding_dsv32 as vendored _MSGS_BASIC = [{"role": "user", "content": "Hello"}] @@ -43,9 +45,7 @@ def _tok_with_model_type(tmp_path, model_type: str) -> _FakeTokenizer: def _reference_encode(messages, *, thinking: bool = False, drop_thinking: bool = True) -> str: """The canonical V3.2 rendering: a direct ``encode_messages`` call. Locks ``render_messages`` to this thin-bridge contract (no preprocessing of its own).""" - from sglang.srt.entrypoints.openai import encoding_dsv32 - - return encoding_dsv32.encode_messages( + return vendored.encode_messages( messages, thinking_mode="thinking" if thinking else "chat", drop_thinking=drop_thinking ) @@ -159,6 +159,114 @@ def test_thinking_mode_changes_output(): ) +# --------------------------------------------------------------------------- +# Miles extension: render-level drop_thinking=False (vendored encoder) +# --------------------------------------------------------------------------- + +_THINKING_HISTORY = [ + {"role": "user", "content": "q1"}, + {"role": "assistant", "content": "a1", "reasoning_content": "r1"}, + {"role": "user", "content": "q2"}, + {"role": "assistant", "content": "a2", "reasoning_content": "r2"}, +] + +_TOOL_TAIL_HISTORY = [ + {"role": "user", "content": "q"}, + { + "role": "assistant", + "content": "", + "reasoning_content": "r", + "tool_calls": [{"type": "function", "function": {"name": "f", "arguments": '{"a": 1}'}}], + }, + {"role": "tool", "content": "out", "tool_call_id": "c0"}, +] + + +def test_drop_thinking_false_renders_historical_thinking(): + dropped = _reference_encode(_THINKING_HISTORY, thinking=True, drop_thinking=True) + kept = _reference_encode(_THINKING_HISTORY, thinking=True, drop_thinking=False) + assert "r1" not in dropped + assert "r1" in kept and "r2" in kept + + +@pytest.mark.parametrize("history", [_THINKING_HISTORY, _TOOL_TAIL_HISTORY], ids=["user-turns", "tool-tail"]) +def test_drop_thinking_false_is_append_only_across_user_append(history): + # The point of the render-level drop_thinking=False extension: a new user + # turn must extend the rendered history byte-for-byte. Upstream's + # last_user_idx gates break this (the tool tail flips its trailing + # to ; earlier assistants lose their thinking block). + before = _reference_encode(history, thinking=True, drop_thinking=False) + after = _reference_encode(history + [{"role": "user", "content": "next"}], thinking=True, drop_thinking=False) + assert after.startswith(before) + + +@pytest.mark.parametrize("history", [_THINKING_HISTORY, _TOOL_TAIL_HISTORY], ids=["user-turns", "tool-tail"]) +def test_drop_thinking_true_is_not_append_only_across_user_append(history): + # Regression guard for why every V3.2 surface pins drop_thinking=False. + before = _reference_encode(history, thinking=True, drop_thinking=True) + after = _reference_encode(history + [{"role": "user", "content": "next"}], thinking=True, drop_thinking=True) + assert not after.startswith(before) + + +def test_tool_only_history_drop_false_matches_drop_true(): + # Pure tool-loop histories (the pre-existing {tool} surface) render + # byte-identically under either drop mode: every assistant sits after the + # single user turn, so the pinned drop_thinking=False changes nothing for + # existing tool-only configs. + history = [ + {"role": "system", "content": "sys"}, + {"role": "user", "content": "q"}, + { + "role": "assistant", + "content": "", + "reasoning_content": "r1", + "tool_calls": [{"type": "function", "function": {"name": "f", "arguments": '{"a": 1}'}}], + }, + {"role": "tool", "content": "out1", "tool_call_id": "c0"}, + { + "role": "assistant", + "content": "", + "reasoning_content": "r2", + "tool_calls": [{"type": "function", "function": {"name": "f", "arguments": '{"a": 2}'}}], + }, + {"role": "tool", "content": "out2", "tool_call_id": "c1"}, + ] + assert _reference_encode(history, thinking=True, drop_thinking=False) == _reference_encode( + history, thinking=True, drop_thinking=True + ) + + +# Upstream parity: every drop_thinking=True path must keep rendering +# byte-identically to the installed sglang encoder the file was vendored from. + +_UPSTREAM_PARITY_SCENARIOS = { + **_PARITY_SCENARIOS, + "developer": [{"role": "developer", "content": "do the thing"}], + "thinking_history": _THINKING_HISTORY, + "tool_tail": _TOOL_TAIL_HISTORY, +} + + +@pytest.mark.parametrize("scenario", list(_UPSTREAM_PARITY_SCENARIOS), ids=list(_UPSTREAM_PARITY_SCENARIOS)) +@pytest.mark.parametrize("thinking", [False, True], ids=["chat", "thinking"]) +def test_drop_thinking_true_matches_upstream_sglang(scenario, thinking): + messages = _UPSTREAM_PARITY_SCENARIOS[scenario] + thinking_mode = "thinking" if thinking else "chat" + assert _reference_encode(messages, thinking=thinking, drop_thinking=True) == upstream.encode_messages( + messages, thinking_mode=thinking_mode, drop_thinking=True + ) + + +def test_drop_thinking_true_raise_parity_with_upstream(): + # thinking mode + a post-last-user assistant without reasoning_content or + # tool_calls raises upstream; the vendored copy keeps that contract. + bad = [{"role": "user", "content": "q"}, {"role": "assistant", "content": "a"}] + with pytest.raises(vendored.DS32EncodingError): + vendored.encode_messages(bad, thinking_mode="thinking", drop_thinking=True) + with pytest.raises(upstream.DS32EncodingError): + upstream.encode_messages(bad, thinking_mode="thinking", drop_thinking=True) + + # --------------------------------------------------------------------------- # Tool injection (sglang-aligned: tools go into the system block) # --------------------------------------------------------------------------- diff --git a/tests/fast/utils/chat_template_utils/test_fixed_templates.py b/tests/fast/utils/chat_template_utils/test_fixed_templates.py index deec6c0d769..7f098911919 100644 --- a/tests/fast/utils/chat_template_utils/test_fixed_templates.py +++ b/tests/fast/utils/chat_template_utils/test_fixed_templates.py @@ -52,6 +52,17 @@ def test_deepseek_v4_tool_user_pins_drop_thinking_false(): assert kwargs == {"drop_thinking": False} +@pytest.mark.parametrize("roles", [["tool"], ["tool", "user"]]) +def test_deepseek_v32_pins_drop_thinking_false_on_every_surface(roles): + # The vendored encoding_dsv32 honors drop_thinking=False at the render + # level (unlike upstream, which strips historical thinking whenever a new + # user turn advances last_user_index). Every V3.2 surface pins it so + # renders stay append-only for both tool and user appends. + path, kwargs = resolve_fixed_chat_template(TITOTokenizerType.DEEPSEEKV32, roles) + assert path is None + assert kwargs == {"drop_thinking": False} + + @pytest.mark.parametrize( "tito_model", [TITOTokenizerType.QWEN3, TITOTokenizerType.QWEN35, TITOTokenizerType.QWENNEXT], diff --git a/tests/fast/utils/chat_template_utils/test_tito_tokenizer.py b/tests/fast/utils/chat_template_utils/test_tito_tokenizer.py index 267e86f19fe..d69e8359f65 100644 --- a/tests/fast/utils/chat_template_utils/test_tito_tokenizer.py +++ b/tests/fast/utils/chat_template_utils/test_tito_tokenizer.py @@ -46,11 +46,15 @@ from __future__ import annotations +import json +from unittest.mock import MagicMock + import pytest from transformers import AutoTokenizer from miles.utils.chat_template_utils import MismatchType, apply_chat_template, resolve_fixed_chat_template from miles.utils.chat_template_utils.tito_tokenizer import ( + DeepSeekV32TITOTokenizer, GLM47TITOTokenizer, Qwen3TITOTokenizer, Qwen35TITOTokenizer, @@ -226,12 +230,91 @@ def test_default(self, default_tito: TITOTokenizer): assert default_tito._assistant_start_str is None assert default_tito.trailing_token_ids == frozenset() + @pytest.mark.parametrize( + "chat_template_kwargs, expected", + [ + pytest.param({}, True, id="default-thinking"), + pytest.param({"enable_thinking": False}, False, id="disable-via-miles-kwarg"), + pytest.param({"thinking": False}, False, id="disable-via-sglang-kwarg"), + pytest.param( + {"enable_thinking": False, "thinking": True}, + False, + id="miles-kwarg-precedes-sglang-kwarg", + ), + pytest.param( + {"thinking_mode": "thinking", "thinking": False}, + True, + id="explicit-mode-precedes-sglang-kwarg", + ), + pytest.param({"thinking_mode": "chat"}, False, id="explicit-chat-mode"), + ], + ) + def test_deepseek_v32_forwards_effective_thinking_mode(self, chat_template_kwargs, expected): + tokenizer = MagicMock() + tokenizer.convert_tokens_to_ids.side_effect = [1, 2] + + tito = DeepSeekV32TITOTokenizer(tokenizer, chat_template_kwargs=chat_template_kwargs) + + assert tito.chat_template_kwargs["thinking"] is expected + def test_comparator_inherits_trailing_ids(self, qwen3_tito: Qwen3TITOTokenizer): """create_comparator propagates trailing_token_ids to the comparator's trim set.""" comp = qwen3_tito.create_comparator() assert comp._trim_trailing_ids == set(qwen3_tito.trailing_token_ids) +class TestDeepSeekV32IncrementalAppend: + """V3.2 rides the default synthetic-prefix suffix diff; with the family's + pinned ``drop_thinking=False`` the vendored encoder renders every turn + position-independently, so the synthetic-prefix incremental must equal the + real-history render suffix for both tool and user appends.""" + + class _CharTokenizer: + def __init__(self, name_or_path: str): + self.name_or_path = name_or_path + + def encode(self, text, add_special_tokens=False): + assert add_special_tokens is False + return [ord(c) for c in text] + + def convert_tokens_to_ids(self, token): + return {"<|User|>": 1, "<|Assistant|>": 2}[token] + + @pytest.mark.parametrize( + "appended", + [ + [{"role": "user", "content": "next question"}], + [{"role": "tool", "content": "out", "tool_call_id": "c0"}], + ], + ids=["user", "tool"], + ) + def test_incremental_equals_real_history_suffix(self, tmp_path, appended): + (tmp_path / "config.json").write_text(json.dumps({"model_type": "deepseek_v32"}), encoding="utf-8") + tokenizer = self._CharTokenizer(str(tmp_path)) + tito = DeepSeekV32TITOTokenizer( + tokenizer, + chat_template_kwargs={"drop_thinking": False}, + allowed_append_roles=["tool", "user"], + ) + old = [ + {"role": "user", "content": "q"}, + { + "role": "assistant", + "content": "", + "reasoning_content": "r", + "tool_calls": [{"type": "function", "function": {"name": "f", "arguments": '{"a": 1}'}}], + }, + ] + new = old + appended + + incremental = tito.tokenize_additional_non_assistant(old, new) + + text_old = tito.apply_chat_template(old, add_generation_prompt=False) + text_new = tito.apply_chat_template(new, add_generation_prompt=True) + assert text_new.startswith(text_old) + assert incremental == tokenizer.encode(text_new[len(text_old) :]) + + # --------------------------------------------------------------------------- # TestMergeTokensBoundary — prefix manipulation with synthetic IDs # diff --git a/tests/fast/utils/test_arguments.py b/tests/fast/utils/test_arguments.py index 2dbec7238f1..4f84cb5dd40 100644 --- a/tests/fast/utils/test_arguments.py +++ b/tests/fast/utils/test_arguments.py @@ -6,6 +6,8 @@ import pytest +from miles.backends.sglang_utils.arguments import add_sglang_arguments +from miles.backends.sglang_utils.arguments import validate_args as validate_sglang_args from miles.utils.arguments import ( _maybe_apply_dumper_overrides, _resolve_ft_components, @@ -288,3 +290,69 @@ def test_enabled_with_components_returns_distinct_copy(self) -> None: assert result == ["train", "rollout"] assert result is not components + + +@pytest.mark.parametrize( + ("parallel_args", "expected"), + [ + ([], (1, 1, 1, 1)), + ( + [ + "--sglang-tensor-parallel-size", + "2", + "--sglang-data-parallel-size", + "3", + "--sglang-pipeline-parallel-size", + "4", + "--sglang-expert-parallel-size", + "5", + "--sglang-enable-dp-attention", + ], + (2, 3, 4, 5), + ), + ( + [ + "--sglang-tp-size", + "2", + "--sglang-dp-size", + "3", + "--sglang-pp-size", + "4", + "--sglang-ep-size", + "5", + "--sglang-enable-dp-attention", + ], + (2, 3, 4, 5), + ), + ], +) +def test_sglang_parallel_sizes_use_short_namespace_fields(parallel_args, expected): + parser = argparse.ArgumentParser() + add_sglang_arguments(parser) + args = parser.parse_args(parallel_args) + + assert (args.sglang_tp_size, args.sglang_dp_size, args.sglang_pp_size, args.sglang_ep_size) == expected + assert not hasattr(args, "sglang_tensor_parallel_size") + assert not hasattr(args, "sglang_data_parallel_size") + assert not hasattr(args, "sglang_pipeline_parallel_size") + assert not hasattr(args, "sglang_expert_parallel_size") + + args.rollout_num_gpus_per_engine = 8 + args.true_on_policy_mode = False + args.recompute_logprobs_via_prefill = False + args.sglang_router_policy = None + args.use_session_server = False + + validate_sglang_args(args) + + assert args.sglang_tp_size == 8 + assert (args.sglang_dp_size, args.sglang_pp_size, args.sglang_ep_size) == expected[1:] + + +def test_sglang_parallel_size_aliases_keep_last_value(): + parser = argparse.ArgumentParser() + add_sglang_arguments(parser) + + args = parser.parse_args(["--sglang-data-parallel-size", "2", "--sglang-dp-size", "3"]) + + assert args.sglang_dp_size == 3 diff --git a/tests/fast/utils/test_utils/test_session_verify_runner.py b/tests/fast/utils/test_utils/test_session_verify_runner.py index 770f7f54018..b2b5b8637d6 100644 --- a/tests/fast/utils/test_utils/test_session_verify_runner.py +++ b/tests/fast/utils/test_utils/test_session_verify_runner.py @@ -55,11 +55,26 @@ def test_namespace_to_train_args_omits_expert_parallel_for_single_expert(): def test_namespace_to_train_args_emits_expert_parallel_for_moe(): - train_args = _build_args(sglang_expert_parallel_size=8) + train_args = _build_args(sglang_ep_size=8) assert "--sglang-expert-parallel-size 8" in train_args +def test_namespace_to_train_args_omits_speculative_decoding_by_default(): + train_args = _build_args() + + assert "--sglang-speculative-" not in train_args + + +def test_namespace_to_train_args_enables_eagle_speculative_decoding(): + train_args = _build_args(enable_spec=True) + + assert "--sglang-speculative-algorithm EAGLE" in train_args + assert "--sglang-speculative-num-steps 2" in train_args + assert "--sglang-speculative-eagle-topk 1" in train_args + assert "--sglang-speculative-num-draft-tokens 3" in train_args + + def _write_metrics(path, entries: list[dict]) -> None: path.write_text("\n".join(json.dumps(entry) for entry in entries) + "\n") diff --git a/tests/manual/tito/__init__.py b/tests/manual/tito/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/manual/tito/deepseek_v32.py b/tests/manual/tito/deepseek_v32.py new file mode 100644 index 00000000000..0b56a7b543a --- /dev/null +++ b/tests/manual/tito/deepseek_v32.py @@ -0,0 +1,23 @@ +import os + +from tests.e2e.sglang.test_session_server_multi_role._common import ModelConfig, run_one + +CONFIG = ModelConfig( + model_name=os.environ.get("DEEPSEEK_V32_MODEL", "deepseek-ai/DeepSeek-V3.2"), + reasoning_parser="deepseek-v3", + tool_call_parser="deepseekv32", + tito_model="deepseekv32", + allowed_append_roles=("tool",), + num_gpus=8, + tp_size=8, + ep_size=8, + enable_spec=True, +) + + +def test_deepseek_v32_session_tito(): + run_one(CONFIG) + + +if __name__ == "__main__": + test_deepseek_v32_session_tito()