diff --git a/README.md b/README.md index f0c24b41..07eb2fc2 100644 --- a/README.md +++ b/README.md @@ -57,7 +57,7 @@ next_prompt_ids = r.bridge_to_next_turn( ) ``` -Hand-coded renderers ship for `qwen3`, `qwen3-vl`, `qwen3.5`, `qwen3.6`, `qwen3.8`, `gemma4`, `glm-5`, `glm-5.1`, `glm-4.5`, `minimax-m2`, `deepseek-v3`, `deepseek-r1`, `kimi-k2`, `kimi-k2.5` / `kimi-k2.6`, `laguna-xs.2`, `laguna-xs-2.1`, `laguna-s-2.1`, `laguna-m.1`, `nemotron-3`, `nemotron-3-ultra`, `nemotron-3.5`, `llama-3`, `gpt-oss`, `hy3`, `inkling` / `inkling-small`, and `prime-qwen3`. Anything else falls back to `DefaultRenderer`, a generic `apply_chat_template` wrapper. `qwen3-vl`, `qwen3.5`, `qwen3.6`, `qwen3.8`, `gemma4`, `kimi-k2.5` / `kimi-k2.6`, and the Inkling checkpoints are multimodal (Inkling handles both image **and** audio). +Hand-coded renderers ship for `qwen3`, `qwen3-vl`, `qwen3.5`, `qwen3.6`, `qwen3.8`, `gemma4`, `glm-5`, `glm-5.1`, `glm-4.5`, `minimax-m2`, `deepseek-v3`, `deepseek-r1`, `deepseek-v4` (V4 Flash 0731), `kimi-k2`, `kimi-k2.5` / `kimi-k2.6`, `laguna-xs.2`, `laguna-xs-2.1`, `laguna-s-2.1`, `laguna-m.1`, `nemotron-3`, `nemotron-3-ultra`, `nemotron-3.5`, `llama-3`, `gpt-oss`, `hy3`, `inkling` / `inkling-small`, and `prime-qwen3`. Anything else falls back to `DefaultRenderer`, a generic `apply_chat_template` wrapper. `qwen3-vl`, `qwen3.5`, `qwen3.6`, `qwen3.8`, `gemma4`, `kimi-k2.5` / `kimi-k2.6`, and the Inkling checkpoints are multimodal (Inkling handles both image **and** audio). ## API diff --git a/docs/renderer-config.md b/docs/renderer-config.md index 91bc62e6..7c996659 100644 --- a/docs/renderer-config.md +++ b/docs/renderer-config.md @@ -51,6 +51,7 @@ definition time. Template fields are covered by parity tests against | Nemotron-3.5 Lightning | `Nemotron35RendererConfig` | `enable_thinking`, `truncate_history_thinking` | - | | DeepSeek V3 | `DeepSeekV3RendererConfig` | - | - | | DeepSeek R1 | `DeepSeekR1RendererConfig` | - | - | +| DeepSeek V4 Flash 0731 | `DeepSeekV4RendererConfig` | `enable_thinking`, `drop_thinking`, `reasoning_effort` | - | Configs are frozen value objects. To override a field, construct a new instance or call `config.model_copy(update={...})`. @@ -145,6 +146,7 @@ the knobs its template actually exposes: | Kimi K2.5 / 2.6 | `thinking=False -> all`, else `tool_cycle` | | Nemotron-3 / 3.5 | `truncate_history_thinking=False -> all`; else `enable_thinking=False -> all`; else `tool_cycle` | | DeepSeek R1 | `template` | +| DeepSeek V4 Flash 0731 | `enable_thinking=False` or `drop_thinking=False -> all`, else `tool_cycle` | | MiniMax M2 | `tool_cycle` | | DeepSeek V3, Qwen3-VL, Kimi K2, Laguna XS.2 / M.1 / XS-2.1 / S-2.1, Llama 3, Inkling | `all` | | PrimeIntellect Qwen3 | `all` | diff --git a/renderers/__init__.py b/renderers/__init__.py index dc9f3696..ae77d19d 100644 --- a/renderers/__init__.py +++ b/renderers/__init__.py @@ -48,6 +48,7 @@ DefaultRendererConfig, DeepSeekR1RendererConfig, DeepSeekV3RendererConfig, + DeepSeekV4RendererConfig, GLM45RendererConfig, GLM51RendererConfig, GLM5RendererConfig, @@ -85,6 +86,7 @@ _LAZY_RENDERERS: dict[str, str] = { "DeepSeekR1Renderer": "renderers.deepseek_r1", "DeepSeekV3Renderer": "renderers.deepseek_v3", + "DeepSeekV4Renderer": "renderers.deepseek_v4", "DefaultRenderer": "renderers.default", "GLM45Renderer": "renderers.glm45", "GLM51Renderer": "renderers.glm5", @@ -138,6 +140,8 @@ def __dir__() -> list[str]: "DeepSeekR1RendererConfig", "DeepSeekV3Renderer", "DeepSeekV3RendererConfig", + "DeepSeekV4Renderer", + "DeepSeekV4RendererConfig", "DefaultRenderer", "DefaultRendererConfig", "GLM45Renderer", diff --git a/renderers/base.py b/renderers/base.py index e98637b0..15aa6d28 100644 --- a/renderers/base.py +++ b/renderers/base.py @@ -982,6 +982,9 @@ def is_multimodal(r: object) -> bool: # DeepSeek R1 (reasoning). "deepseek-ai/DeepSeek-R1": "deepseek-r1", "deepseek-ai/DeepSeek-R1-0528": "deepseek-r1", + # DeepSeek V4 Flash 0731 uses the repository's Python DSML encoder (the + # tokenizer intentionally ships no Jinja chat_template). + "deepseek-ai/DeepSeek-V4-Flash-0731": "deepseek-v4", # Kimi K2 (K2.5 and K2.6 share the K2.5 template, distinct from K2). "moonshotai/Kimi-K2-Instruct": "kimi-k2", "moonshotai/Kimi-K2.5": "kimi-k2.5", @@ -1299,6 +1302,7 @@ def _populate_registry(): return from renderers.deepseek_r1 import DeepSeekR1Renderer from renderers.deepseek_v3 import DeepSeekV3Renderer + from renderers.deepseek_v4 import DeepSeekV4Renderer from renderers.default import DefaultRenderer from renderers.glm5 import GLM5Renderer, GLM51Renderer from renderers.glm45 import GLM45Renderer @@ -1344,6 +1348,7 @@ def _populate_registry(): "minimax-m2": MiniMaxM2Renderer, "deepseek-v3": DeepSeekV3Renderer, "deepseek-r1": DeepSeekR1Renderer, + "deepseek-v4": DeepSeekV4Renderer, "hy3": Hy3Renderer, "inkling": InklingRenderer, "kimi-k2": KimiK2Renderer, diff --git a/renderers/configs.py b/renderers/configs.py index f257b6b1..94592a66 100644 --- a/renderers/configs.py +++ b/renderers/configs.py @@ -930,6 +930,47 @@ class DeepSeekR1RendererConfig(BaseRendererConfig): _template_fields = frozenset() +class DeepSeekV4RendererConfig(BaseRendererConfig): + """DeepSeek-V4-Flash-0731 reference-encoder configuration. + + The checkpoint ships a Python encoder rather than a Jinja template. These + fields mirror its public controls: chat vs thinking mode, historical + reasoning dropping, and the opt-in thinking-effort prefix. + """ + + name: Literal["deepseek-v4"] = "deepseek-v4" + _template_fields = frozenset( + {"enable_thinking", "drop_thinking", "reasoning_effort"} + ) + + enable_thinking: bool = False + """Select thinking mode. ``False`` matches the official inference script.""" + + drop_thinking: bool = True + """Drop reasoning before the latest user query when no tools are present. + + The reference encoder automatically preserves all reasoning whenever tools + are supplied, regardless of this value. + """ + + reasoning_effort: Literal["low", "high", "max"] = "low" + """Thinking-only effort prefix; ``low`` adds no text. + + ``low`` is the checkpoint Python encoder's default. DeepSeek's hosted API + independently defaults its thinking effort to ``high``. + """ + + @model_validator(mode="after") + def _check_thinking_retention(self): + _reject_thinking_retention_conflict( + self, + "drop_thinking", + true_implies="tool_cycle", + false_implies="all", + ) + return self + + RendererConfig = Annotated[ Union[ AutoRendererConfig, @@ -960,6 +1001,7 @@ class DeepSeekR1RendererConfig(BaseRendererConfig): Nemotron35RendererConfig, DeepSeekV3RendererConfig, DeepSeekR1RendererConfig, + DeepSeekV4RendererConfig, ], Field(discriminator="name"), ] @@ -1006,6 +1048,7 @@ class DeepSeekR1RendererConfig(BaseRendererConfig): "nemotron-3.5": Nemotron35RendererConfig, "deepseek-v3": DeepSeekV3RendererConfig, "deepseek-r1": DeepSeekR1RendererConfig, + "deepseek-v4": DeepSeekV4RendererConfig, } @@ -1039,6 +1082,7 @@ def config_from_name(name: str) -> BaseRendererConfig | None: "DefaultRendererConfig", "DeepSeekR1RendererConfig", "DeepSeekV3RendererConfig", + "DeepSeekV4RendererConfig", "GLM45RendererConfig", "GLM51RendererConfig", "GLM5RendererConfig", diff --git a/renderers/deepseek_v4.py b/renderers/deepseek_v4.py new file mode 100644 index 00000000..e5fadc84 --- /dev/null +++ b/renderers/deepseek_v4.py @@ -0,0 +1,859 @@ +"""DeepSeek V4 Flash 0731 renderer. + +The checkpoint does not ship a Jinja chat template. Its source of truth is +``encoding/encoding_dsv4.py`` in the model repository. This module mirrors +that encoder while adapting it to the renderer protocol: + +* ``enable_thinking=False`` selects the reference encoder's ``chat`` mode; + ``True`` selects ``thinking`` mode. +* tools are accepted through :meth:`render` and injected on the first + developer message, otherwise the first system message (the two locations + supported by the reference encoder). +* OpenAI ``tool`` messages are merged into a DeepSeek user turn as + ```` blocks and parallel results are sorted by call order. +* DSML tool calls are parsed back to :class:`ParsedToolCall` records. + +Special-token spelling matters: ``｜`` is U+FF5C and ``▁`` is U+2581. +""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import Any, Mapping + +from renderers.base import ( + Message, + ParsedResponse, + RenderedTokens, + Tokenizer, + ToolSpec, + _content_mask_or_empty, + _get_offset_tokenizer, + _infer_offsets_from_decode, + extract_message_tool_names, + reject_assistant_in_extension, + resolve_thinking_retention, + should_rerender_for_thinking_retention, + trim_to_turn_close, +) +from renderers.configs import DeepSeekV4RendererConfig +from renderers.parsing import parse_deepseek_v4 + + +_BOS = "<｜begin▁of▁sentence｜>" +_EOS = "<｜end▁of▁sentence｜>" +_USER = "<｜User｜>" +_ASSISTANT = "<｜Assistant｜>" +_LATEST_REMINDER = "<｜latest_reminder｜>" +_THINK_START = "" +_THINK_END = "" +_DSML = "｜DSML｜" +_QUERY_ROLES = frozenset({"user", "developer"}) + +_TASK_TOKENS = { + "action": "<｜action｜>", + "query": "<｜query｜>", + "authority": "<｜authority｜>", + "domain": "<｜domain｜>", + "title": "<｜title｜>", + "read_url": "<｜read_url｜>", +} + +_REASONING_EFFORT_PROMPTS = { + "low": "", + "high": ( + "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" + "You MUST be very thorough in your thinking and comprehensively " + "decompose the problem to resolve the root cause, rigorously " + "stress-testing your logic against all potential paths, edge cases, " + "and adversarial scenarios.\n" + "Explicitly write out your entire deliberation process, documenting " + "every intermediate step, considered alternative, and rejected " + "hypothesis to ensure absolutely no assumption is left unchecked.\n\n" + ), + "max": ( + "Reasoning Effort: Beyond maximum — exhaustive, relentless, and " + "uncompromising.\n" + "You MUST reason with the utmost depth and rigor, leaving absolutely " + "nothing to chance: exhaustively decompose the problem into its most " + "fundamental components, trace every causal chain to its root, and " + "resolve the underlying cause rather than any surface symptom.\n" + "Do not stop reasoning until you have independently verified the " + "solution from multiple angles and are certain that no assumption " + "remains unchecked and no error remains undiscovered.\n\n" + ), +} + +_TOOLS_TEMPLATE = """## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml}tool_calls>" block like the following: + +<{dsml}tool_calls> +<{dsml}invoke name="$TOOL_NAME"> +<{dsml}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml}invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by {think_start}), you MUST output your complete reasoning inside {think_start}...{think_end} BEFORE any tool calls or final response. + +Otherwise, output directly after {think_end} with tool calls or final response. + +### Available Tool Schemas + +{tool_schemas} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +""" + + +@dataclass +class _ContentBlock: + kind: str + content: str + message_index: int + tool_call_id: str = "" + + +@dataclass +class _LogicalMessage: + role: str + message_index: int + content: str = "" + blocks: list[_ContentBlock] = field(default_factory=list) + tool_calls: list[Mapping[str, Any]] = field(default_factory=list) + reasoning_content: str = "" + task: str | None = None + wo_eos: bool = False + response_format: Any = None + + +def _json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False) + + +def _text_content(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if not isinstance(value, list): + return str(value) + + parts: list[str] = [] + for part in value: + if not isinstance(part, Mapping): + continue + part_type = part.get("type") + if part_type == "text": + parts.append(str(part.get("text", ""))) + elif part_type == "thinking": + # Structured thinking belongs in ``reasoning_content``. Ignore it + # in visible content rather than leaking it into the answer. + continue + else: + parts.append(f"[Unsupported {part_type}]") + return "\n\n".join(parts) + + +def _tool_result_content(value: Any) -> str: + if not isinstance(value, list): + return "" if value is None else str(value) + parts: list[str] = [] + for part in value: + if isinstance(part, Mapping) and part.get("type") == "text": + parts.append(str(part.get("text", ""))) + elif isinstance(part, Mapping): + parts.append(f"[Unsupported {part.get('type')}]") + else: + parts.append(f"[Unsupported {type(part).__name__}]") + return "\n\n".join(parts) + + +def _reasoning_content(message: Mapping[str, Any]) -> str: + reasoning = message.get("reasoning_content") + if reasoning is None: + reasoning = message.get("reasoning") + if reasoning is not None: + return str(reasoning) + content = message.get("content") + if isinstance(content, list): + return "".join( + str(part.get("thinking", "")) + for part in content + if isinstance(part, Mapping) and part.get("type") == "thinking" + ) + return "" + + +def _tool_function(tool: Mapping[str, Any]) -> Mapping[str, Any]: + function = tool.get("function") + return function if isinstance(function, Mapping) else tool + + +def _tool_call_function(tool_call: Mapping[str, Any]) -> Mapping[str, Any]: + function = tool_call.get("function") + return function if isinstance(function, Mapping) else tool_call + + +def _is_query_message(message: Message) -> bool: + """Match the reference encoder's user/developer query boundary.""" + return message.get("role") in _QUERY_ROLES + + +def _prepare_messages(messages: list[Message]) -> list[_LogicalMessage]: + """Apply DeepSeek's user/tool merge and parallel-result ordering.""" + merged: list[_LogicalMessage] = [] + + for index, message in enumerate(messages): + role = str(message.get("role") or "") + if role == "tool": + block = _ContentBlock( + kind="tool_result", + content=_tool_result_content(message.get("content")), + message_index=index, + tool_call_id=str(message.get("tool_call_id") or ""), + ) + if merged and merged[-1].role == "user": + merged[-1].blocks.append(block) + else: + merged.append( + _LogicalMessage( + role="user", + message_index=index, + blocks=[block], + ) + ) + continue + + if role == "user": + block = _ContentBlock( + kind="text", + content=_text_content(message.get("content")), + message_index=index, + ) + if merged and merged[-1].role == "user" and merged[-1].task is None: + merged[-1].blocks.append(block) + else: + merged.append( + _LogicalMessage( + role="user", + message_index=index, + content=block.content, + blocks=[block], + task=message.get("task"), + wo_eos=bool(message.get("wo_eos", False)), + ) + ) + continue + + logical = _LogicalMessage( + role=role, + message_index=index, + content=_text_content(message.get("content")), + task=message.get("task"), + wo_eos=bool(message.get("wo_eos", False)), + response_format=message.get("response_format"), + ) + if role == "assistant": + logical.reasoning_content = _reasoning_content(message) + logical.tool_calls = list(message.get("tool_calls") or []) + merged.append(logical) + + call_order: dict[str, int] = {} + for message in merged: + if message.role == "assistant" and message.tool_calls: + call_order = {} + for index, tool_call in enumerate(message.tool_calls): + function = _tool_call_function(tool_call) + call_id = tool_call.get("id") or function.get("id") + if call_id: + call_order[str(call_id)] = index + elif message.role == "user": + result_blocks = [b for b in message.blocks if b.kind == "tool_result"] + if len(result_blocks) < 2 or not call_order: + continue + result_blocks.sort(key=lambda b: call_order.get(b.tool_call_id, 0)) + ordered = iter(result_blocks) + message.blocks = [ + next(ordered) if block.kind == "tool_result" else block + for block in message.blocks + ] + + return merged + + +class DeepSeekV4Renderer: + """Renderer for ``deepseek-ai/DeepSeek-V4-Flash-0731``.""" + + _implied_thinking_retention = "tool_cycle" + + def __init__( + self, + tokenizer: Tokenizer, + config: DeepSeekV4RendererConfig | None = None, + ): + self._tokenizer = tokenizer + self.config = config or DeepSeekV4RendererConfig() + implied_retention = ( + "tool_cycle" + if self.config.enable_thinking and self.config.drop_thinking + else "all" + ) + self.effective_thinking_retention = resolve_thinking_retention( + self.config, + implied_retention, + ) + + self._bos = self._special_id(_BOS) + self._eos = self._special_id(_EOS) + self._user = self._special_id(_USER) + self._assistant = self._special_id(_ASSISTANT) + self._latest_reminder = self._special_id(_LATEST_REMINDER) + self._think_start = self._special_id(_THINK_START) + self._think_end = self._special_id(_THINK_END) + self._dsml = self._special_id(_DSML) + self._task_ids = { + name: self._special_id(token) for name, token in _TASK_TOKENS.items() + } + + def _special_id(self, token: str) -> int: + ids = self._tokenizer.encode(token, add_special_tokens=False) + if len(ids) != 1: + raise ValueError(f"Expected one token for {token!r}, got {ids}") + return ids[0] + + @staticmethod + def _render_tools(tools: list[ToolSpec]) -> str: + schemas = [_json(dict(_tool_function(tool))) for tool in tools] + return _TOOLS_TEMPLATE.format( + dsml=_DSML, + think_start=_THINK_START, + think_end=_THINK_END, + tool_schemas="\n".join(schemas), + ) + + @staticmethod + def _render_tool_call(tool_call: Mapping[str, Any]) -> str: + function = _tool_call_function(tool_call) + name = function.get("name") + raw_arguments = function.get("arguments", {}) + if isinstance(raw_arguments, str): + try: + arguments = json.loads(raw_arguments) + except (json.JSONDecodeError, TypeError): + arguments = {"arguments": raw_arguments} + elif isinstance(raw_arguments, Mapping): + arguments = dict(raw_arguments) + else: + arguments = {"arguments": raw_arguments} + + params: list[str] = [] + for key, value in arguments.items(): + is_string = isinstance(value, str) + rendered_value = value if is_string else _json(value) + params.append( + f'<{_DSML}parameter name="{key}" ' + f'string="{str(is_string).lower()}">{rendered_value}' + f"" + ) + arguments_text = "\n".join(params) + return f'<{_DSML}invoke name="{name}">\n{arguments_text}\n' + + def render( + self, + messages: list[Message], + *, + tools: list[ToolSpec] | None = None, + add_generation_prompt: bool = False, + ) -> RenderedTokens: + return self._render( + messages, + tools=tools, + add_generation_prompt=add_generation_prompt, + add_bos=True, + add_effort_prompt=True, + ) + + def _render( + self, + messages: list[Message], + *, + tools: list[ToolSpec] | None, + add_generation_prompt: bool, + add_bos: bool, + add_effort_prompt: bool, + ) -> RenderedTokens: + if not messages: + raise ValueError("No messages provided.") + + logical_messages = _prepare_messages(messages) + if not logical_messages: + raise ValueError("No renderable messages provided.") + + effective_drop_thinking = self.config.drop_thinking and not tools + if self.config.enable_thinking and effective_drop_thinking: + last_query = max( + ( + index + for index, message in enumerate(logical_messages) + if message.role in _QUERY_ROLES + ), + default=-1, + ) + # The reference encoder removes internal developer/search-agent + # messages before the latest query when historical thinking is + # dropped. Public tool flows retain them because tools force + # ``effective_drop_thinking=False``. + logical_messages = [ + message + for index, message in enumerate(logical_messages) + if message.role != "developer" or index >= last_query + ] + + token_ids: list[int] = [] + message_indices: list[int] = [] + sampled_mask: list[bool] = [] + is_content: list[bool] = [] + pending_text: list[tuple[str, int, bool, bool]] = [] + + def emit_ids( + ids: list[int], + message_index: int, + *, + sampled: bool = False, + content: bool = False, + ) -> None: + token_ids.extend(ids) + message_indices.extend([message_index] * len(ids)) + sampled_mask.extend([sampled] * len(ids)) + is_content.extend([content] * len(ids)) + + def flush_text() -> None: + """Tokenize contiguous text once, preserving source metadata. + + The official encoder builds one prompt string. Encoding renderer + fragments independently can therefore change BPE merges at their + boundaries even when the decoded text is identical. Offset maps + let us recover message/sample/content attribution after the + required single encoding pass. + """ + if not pending_text: + return + + full_text = "".join(text for text, *_ in pending_text) + if not full_text: + pending_text.clear() + return + + spans: list[tuple[int, int, int, bool, bool]] = [] + position = 0 + for text, message_index, sampled, content in pending_text: + end = position + len(text) + if end > position: + spans.append((position, end, message_index, sampled, content)) + position = end + + offset_tokenizer = _get_offset_tokenizer(self._tokenizer) + if offset_tokenizer is None: + text_ids = self._tokenizer.encode( + full_text, + add_special_tokens=False, + ) + offsets = _infer_offsets_from_decode( + self._tokenizer, + text_ids, + full_text, + ) + has_content_attribution = False + if offsets is None: + # Token IDs remain exact even when a lossy decoder makes + # source boundaries unrecoverable. Text runs separated by + # special tokens have one sampled state, so retain that + # signal and associate the opaque run with a contributing + # caller message. + fallback_message_index = next( + ( + span_message_index + for _, _, span_message_index, _, _ in spans + if span_message_index >= 0 + ), + spans[-1][2] if spans else -1, + ) + fallback_sampled = spans[-1][3] if spans else False + emit_ids( + text_ids, + fallback_message_index, + sampled=fallback_sampled, + ) + pending_text.clear() + return + else: + encoding = offset_tokenizer( + full_text, + add_special_tokens=False, + return_offsets_mapping=True, + ) + text_ids = list(encoding["input_ids"]) + offsets = list(encoding["offset_mapping"]) + has_content_attribution = True + + fallback = spans[-1][2:] if spans else (-1, False, False) + for token_id, (start, end) in zip(text_ids, offsets): + metadata: tuple[int, bool, bool] = fallback + for ( + span_start, + span_end, + span_message_index, + span_sampled, + span_content, + ) in spans: + if span_start <= start < span_end: + metadata = ( + span_message_index, + span_sampled, + span_content, + ) + break + + message_index, sampled, content = metadata + if has_content_attribution and end > start: + # Preserve every body byte when a BPE token straddles a + # scaffold/content boundary. This intentionally permits a + # few adjacent scaffold bytes to share the content bit. + content = any( + span_content + for span_start, span_end, _, _, span_content in spans + if span_start < end and start < span_end + ) + elif not has_content_attribution: + content = False + emit_ids( + [token_id], + message_index, + sampled=sampled, + content=content, + ) + + pending_text.clear() + + def emit_special( + token_id: int, + message_index: int, + *, + sampled: bool = False, + content: bool = False, + ) -> None: + flush_text() + emit_ids( + [token_id], + message_index, + sampled=sampled, + content=content, + ) + + def emit_text( + text: str, + message_index: int, + *, + sampled: bool = False, + content: bool = False, + ) -> None: + if text: + pending_text.append((text, message_index, sampled, content)) + + if add_bos: + emit_special(self._bos, -1) + if add_effort_prompt and self.config.enable_thinking: + emit_text( + _REASONING_EFFORT_PROMPTS[self.config.reasoning_effort], + -1, + ) + + last_query_index = -1 + for index, message in enumerate(logical_messages): + if message.role in _QUERY_ROLES: + last_query_index = index + + tool_target: int | None = None + if tools: + tool_target = next( + ( + index + for index, message in enumerate(logical_messages) + if message.role == "developer" + ), + None, + ) + if tool_target is None: + tool_target = next( + ( + index + for index, message in enumerate(logical_messages) + if message.role == "system" + ), + None, + ) + if tool_target is None: + # Equivalent to an empty synthetic system message carrying + # tools in the reference encoder. + emit_text("\n\n" + self._render_tools(tools), -1) + + for index, message in enumerate(logical_messages): + role = message.role + msg_idx = message.message_index + + if role == "system": + emit_text(message.content, msg_idx, content=True) + + elif role == "developer": + if not message.content: + raise ValueError("Developer messages require content.") + emit_special(self._user, msg_idx) + emit_text(message.content, msg_idx, content=True) + + elif role == "user": + emit_special(self._user, msg_idx) + for block_index, block in enumerate(message.blocks): + if block_index: + emit_text("\n\n", block.message_index) + if block.kind == "tool_result": + emit_text("", block.message_index) + emit_text( + block.content, + block.message_index, + content=True, + ) + emit_text("", block.message_index) + else: + emit_text( + block.content, + block.message_index, + content=True, + ) + + elif role == "latest_reminder": + emit_special(self._latest_reminder, msg_idx) + emit_text(message.content, msg_idx, content=True) + + elif role == "assistant": + previous_has_task = ( + index > 0 and logical_messages[index - 1].task is not None + ) + keep_reasoning = ( + self.config.enable_thinking + and not previous_has_task + and (not effective_drop_thinking or index > last_query_index) + ) + if keep_reasoning: + emit_text( + message.reasoning_content, + msg_idx, + sampled=True, + content=True, + ) + emit_special( + self._think_end, + msg_idx, + sampled=True, + content=True, + ) + + emit_text( + message.content, + msg_idx, + sampled=True, + content=True, + ) + if message.tool_calls: + rendered_calls = "\n".join( + self._render_tool_call(call) for call in message.tool_calls + ) + emit_text( + f"\n\n<{_DSML}tool_calls>\n{rendered_calls}\n" + f"", + msg_idx, + sampled=True, + content=True, + ) + if not message.wo_eos: + emit_special( + self._eos, + msg_idx, + sampled=True, + content=True, + ) + + else: + raise ValueError(f"Unsupported DeepSeek V4 role: {role!r}") + + if tools and index == tool_target: + emit_text("\n\n" + self._render_tools(tools), msg_idx) + if message.response_format is not None and role in { + "system", + "developer", + }: + emit_text( + "\n\n## Response Format:\n\n" + "You MUST strictly adhere to the following schema to reply:\n" + + _json(message.response_format), + msg_idx, + ) + + next_role = ( + logical_messages[index + 1].role + if index + 1 < len(logical_messages) + else None + ) + transition_needed = next_role in {"assistant", "latest_reminder"} + if next_role is None: + transition_needed = message.task is not None or add_generation_prompt + if not transition_needed: + continue + + task = message.task + transition_msg_idx = -1 + if next_role is not None: + for following in logical_messages[index + 1 :]: + if following.role == "assistant": + transition_msg_idx = following.message_index + break + + if task is not None: + if task not in self._task_ids: + raise ValueError( + f"Invalid DeepSeek V4 task {task!r}; expected one of " + f"{sorted(self._task_ids)}" + ) + if task != "action": + emit_special(self._task_ids[task], transition_msg_idx) + continue + emit_special(self._assistant, transition_msg_idx) + emit_special( + self._think_start + if self.config.enable_thinking + else self._think_end, + transition_msg_idx, + ) + emit_special(self._task_ids[task], transition_msg_idx) + continue + + if role not in _QUERY_ROLES: + continue + emit_special(self._assistant, transition_msg_idx) + open_thinking = self.config.enable_thinking and ( + not effective_drop_thinking or index >= last_query_index + ) + emit_special( + self._think_start if open_thinking else self._think_end, + transition_msg_idx, + ) + + flush_text() + return RenderedTokens( + token_ids=token_ids, + message_indices=message_indices, + sampled_mask=sampled_mask, + is_content=_content_mask_or_empty(self._tokenizer, is_content), + message_roles=[message.get("role") or "" for message in messages], + message_tool_names=extract_message_tool_names(messages), + ) + + def render_ids( + self, + messages: list[Message], + *, + tools: list[ToolSpec] | None = None, + add_generation_prompt: bool = False, + ) -> list[int]: + return self.render( + messages, + tools=tools, + add_generation_prompt=add_generation_prompt, + ).token_ids + + def parse_response( + self, + token_ids: list[int], + *, + tools: list[ToolSpec] | None = None, # noqa: ARG002 + ) -> ParsedResponse: + return parse_deepseek_v4( + self._tokenizer, + token_ids, + stop_ids={self._eos}, + thinking_enabled=self.config.enable_thinking, + think_end_id=self._think_end, + dsml_id=self._dsml, + ) + + def get_stop_token_ids(self) -> list[int]: + return [self._eos] + + def bridge_to_next_turn( + self, + previous_prompt_ids: list[int], + previous_completion_ids: list[int], + new_messages: list[Message], + *, + tools: list[ToolSpec] | None = None, # noqa: ARG002 + ) -> RenderedTokens | None: + if ( + not previous_prompt_ids + or not new_messages + or reject_assistant_in_extension(new_messages) + ): + return None + if should_rerender_for_thinking_retention( + self.effective_thinking_retention, + new_messages, + is_user_query=_is_query_message, + ): + return None + # Full rendering sorts parallel tool results by IDs from the issuing + # assistant. The bridge only receives the new slice, so it cannot + # prove parity when more than one result is present. + if sum(message.get("role") == "tool" for message in new_messages) > 1: + return None + + previous_ids = trim_to_turn_close( + previous_prompt_ids, + previous_completion_ids, + {self._eos}, + synthesize_close=self._eos, + ) + if previous_ids is None: + return None + + try: + extension = self._render( + new_messages, + tools=None, + add_generation_prompt=True, + add_bos=False, + add_effort_prompt=False, + ) + except ValueError: + return None + + prior_length = len(previous_ids) + return RenderedTokens( + token_ids=previous_ids + extension.token_ids, + message_indices=[-1] * prior_length + extension.message_indices, + sampled_mask=[False] * (prior_length + len(extension.token_ids)), + is_content=_content_mask_or_empty( + self._tokenizer, + [False] * prior_length + extension.is_content, + ), + message_roles=extension.message_roles, + message_tool_names=extension.message_tool_names, + ) + + +__all__ = ["DeepSeekV4Renderer"] diff --git a/renderers/parsing.py b/renderers/parsing.py index f07fdbe5..513eb67a 100644 --- a/renderers/parsing.py +++ b/renderers/parsing.py @@ -1149,6 +1149,196 @@ def _parse_deepseek_tool_calls( return tool_calls +# ── DeepSeek V4: DSML tool calls + single-token ──────────── + + +def parse_deepseek_v4( + tokenizer, + token_ids: list[int], + *, + stop_ids: set[int], + thinking_enabled: bool, + think_end_id: int, + dsml_id: int, +) -> ParsedResponse: + """Parse a DeepSeek V4 completion. + + Thinking mode prefills ```` in the prompt, so the completion starts + with reasoning and closes it with the single-token ````. Tool + calls use DSML markup; the ``｜DSML｜`` marker is a special token, and the + decoded grammar carries an explicit ``string=`` flag for lossless argument + type recovery. + """ + ids = _strip_stop_tokens(token_ids, stop_ids) + + reasoning: str | None = None + content_offset = 0 + if thinking_enabled: + think_end = _find(ids, think_end_id) + if think_end == -1: + return ParsedResponse( + content="", + reasoning_content=_decode(tokenizer, ids) or None, + tool_calls=[], + ) + reasoning = _decode(tokenizer, ids[:think_end]) + content_offset = think_end + 1 + + content_ids = ids[content_offset:] + decoded = _decode(tokenizer, content_ids) + section_marker = "\n\n<｜DSML｜tool_calls>" + section_pos = decoded.find(section_marker) + if section_pos == -1 or dsml_id not in content_ids: + return ParsedResponse( + content=decoded, + reasoning_content=reasoning or None, + tool_calls=[], + ) + + content = decoded[:section_pos] + section_text = decoded[section_pos:] + section_token_offset = content_offset + _decoded_char_to_token_index( + tokenizer, + content_ids, + section_pos, + ) + tool_calls = _parse_deepseek_v4_tool_calls( + tokenizer, + section_text, + ids[section_token_offset:], + section_offset=section_token_offset, + ) + return ParsedResponse( + content=content, + reasoning_content=reasoning or None, + tool_calls=tool_calls, + ) + + +def _decoded_char_to_token_index(tokenizer, ids: list[int], char_index: int) -> int: + """Return the first token boundary at or beyond a decoded char offset.""" + if char_index <= 0: + return 0 + for boundary in range(1, len(ids) + 1): + if len(_decode(tokenizer, ids[:boundary])) >= char_index: + return boundary + return len(ids) + + +def _parse_deepseek_v4_tool_calls( + tokenizer, + section_text: str, + section_ids: list[int], + *, + section_offset: int, +) -> list[ParsedToolCall]: + """Parse every DSML ``invoke`` attempt from one tool-calls section.""" + import re + + invoke_start = '<｜DSML｜invoke name="' + invoke_end = "" + section_end = "" + parameter_pattern = re.compile( + r'<｜DSML｜parameter name="(.*?)" string="(true|false)">' + r"(.*?)", + re.DOTALL, + ) + + tool_calls: list[ParsedToolCall] = [] + outer_end = section_text.find(section_end) + for invoke_match in re.finditer(r"<｜DSML｜invoke", section_text): + start = invoke_match.start() + if outer_end != -1 and outer_end < start: + break + + close = section_text.find(invoke_end, start + len(invoke_start)) + unclosed = close == -1 or (outer_end != -1 and outer_end < close) + block_end = outer_end if unclosed and outer_end != -1 else len(section_text) + if not unclosed: + block_end = close + len(invoke_end) + raw = section_text[start:block_end] + + token_start = _decoded_char_to_token_index(tokenizer, section_ids, start) + token_end = _decoded_char_to_token_index(tokenizer, section_ids, block_end) + span = (section_offset + token_start, section_offset + token_end) + + header_end = section_text.find(">\n", start, block_end) + name: str | None = None + malformed = False + if header_end == -1: + malformed = True + body = "" + else: + header = section_text[start : header_end + 2] + name_match = re.fullmatch( + r'<｜DSML｜invoke name="(.*?)">\n', + header, + flags=re.DOTALL, + ) + if name_match: + name = name_match.group(1) + else: + malformed = True + body_end = close if not unclosed else block_end + body = section_text[header_end + 2 : body_end] + + arguments: dict[str, Any] = {} + invalid_json = False + matched_ranges: list[tuple[int, int]] = [] + for match in parameter_pattern.finditer(body): + key, is_string, raw_value = match.groups() + matched_ranges.append(match.span()) + if key in arguments: + malformed = True + continue + if is_string == "true": + arguments[key] = raw_value + else: + try: + arguments[key] = json.loads(raw_value) + except (json.JSONDecodeError, ValueError): + arguments[key] = raw_value + invalid_json = True + + remainder_parts: list[str] = [] + previous_end = 0 + for match_start, match_end in matched_ranges: + remainder_parts.append(body[previous_end:match_start]) + previous_end = match_end + remainder_parts.append(body[previous_end:]) + remainder = "".join(remainder_parts) + # The canonical form has one newline before ```` and newlines + # between parameters. Anything else left after removing parameter + # blocks is structural debris. + if remainder.strip("\n"): + malformed = True + + if unclosed: + status = ToolCallParseStatus.UNCLOSED_BLOCK + elif not name: + status = ToolCallParseStatus.MISSING_NAME + elif malformed: + status = ToolCallParseStatus.MALFORMED_STRUCTURE + elif invalid_json: + status = ToolCallParseStatus.INVALID_JSON + else: + status = ToolCallParseStatus.OK + + tool_calls.append( + ParsedToolCall( + raw=raw, + name=name, + arguments=arguments, + token_span=span, + status=status, + ) + ) + if unclosed: + break + + return tool_calls + + # ── MiniMax: ... ──────────── diff --git a/tests/conftest.py b/tests/conftest.py index 92d36e64..7616c26e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,6 +61,11 @@ # there's just no byte-output to parity-check against. Split-specific # parity (V3 bare prompt vs R1 +history-strip) is covered in # tests/test_deepseek_r1.py. + # DeepSeek V4 *is* in the shared barrage: unlike V3/R1, its official + # Python encoder covers every shared tool shape. The model-aware oracle in + # tests/reference_rendering.py routes it through that encoder instead of + # tokenizer.apply_chat_template (the V4 tokenizer intentionally has none). + ("deepseek-ai/DeepSeek-V4-Flash-0731", "auto"), # Llama-3 uses the canonical Meta ID for renderer auto-resolution, while # load_tokenizer fetches the tokenizer/chat_template from the unrestricted # unsloth mirror so CI needs no Meta-gated HF token. diff --git a/tests/reference_rendering.py b/tests/reference_rendering.py new file mode 100644 index 00000000..a38255ec --- /dev/null +++ b/tests/reference_rendering.py @@ -0,0 +1,414 @@ +"""Model-aware reference rendering for the shared test barrage. + +Most checkpoints use Hugging Face ``apply_chat_template`` as their source of +truth. DeepSeek V4 Flash 0731 intentionally ships no Jinja template; its model +repository defines the prompt contract in ``encoding/encoding_dsv4.py``. +``render_reference`` hides that distinction so every renderer can run through +the same parity cases. + +The compact DSV4 implementation below is deliberately test-side and independent +of ``renderers.deepseek_v4``. It mirrors the public chat/tool branches of the +official encoder needed by the shared barrage; the official repository fixtures +remain covered separately in ``test_deepseek_v4.py``. +""" + +from __future__ import annotations + +import copy +import json +from collections.abc import Mapping +from typing import Any + + +DEEPSEEK_V4_MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" + +_BOS = "<｜begin▁of▁sentence｜>" +_EOS = "<｜end▁of▁sentence｜>" +_USER = "<｜User｜>" +_ASSISTANT = "<｜Assistant｜>" +_LATEST_REMINDER = "<｜latest_reminder｜>" +_THINK_START = "" +_THINK_END = "" +_DSML = "｜DSML｜" + +_TASK_TOKENS = { + "action": "<｜action｜>", + "query": "<｜query｜>", + "authority": "<｜authority｜>", + "domain": "<｜domain｜>", + "title": "<｜title｜>", + "read_url": "<｜read_url｜>", +} + +_REASONING_EFFORT_PROMPTS = { + "low": "", + "high": ( + "Reasoning Effort: Absolute maximum with no shortcuts permitted.\n" + "You MUST be very thorough in your thinking and comprehensively " + "decompose the problem to resolve the root cause, rigorously " + "stress-testing your logic against all potential paths, edge cases, " + "and adversarial scenarios.\n" + "Explicitly write out your entire deliberation process, documenting " + "every intermediate step, considered alternative, and rejected " + "hypothesis to ensure absolutely no assumption is left unchecked.\n\n" + ), + "max": ( + "Reasoning Effort: Beyond maximum — exhaustive, relentless, and " + "uncompromising.\n" + "You MUST reason with the utmost depth and rigor, leaving absolutely " + "nothing to chance: exhaustively decompose the problem into its most " + "fundamental components, trace every causal chain to its root, and " + "resolve the underlying cause rather than any surface symptom.\n" + "Do not stop reasoning until you have independently verified the " + "solution from multiple angles and are certain that no assumption " + "remains unchecked and no error remains undiscovered.\n\n" + ), +} + +_TOOLS_TEMPLATE = """## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml}tool_calls>" block like the following: + +<{dsml}tool_calls> +<{dsml}invoke name="$TOOL_NAME"> +<{dsml}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml}invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by {think_start}), you MUST output your complete reasoning inside {think_start}...{think_end} BEFORE any tool calls or final response. + +Otherwise, output directly after {think_end} with tool calls or final response. + +### Available Tool Schemas + +{tool_schemas} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +""" + + +def _json(value: Any) -> str: + return json.dumps(value, ensure_ascii=False) + + +def _function(spec: Mapping[str, Any]) -> Mapping[str, Any]: + function = spec.get("function") + return function if isinstance(function, Mapping) else spec + + +def _text(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if not isinstance(value, list): + return str(value) + return "\n\n".join( + str(part.get("text", "")) + for part in value + if isinstance(part, Mapping) and part.get("type") == "text" + ) + + +def _render_tools(tools: list[dict[str, Any]]) -> str: + return _TOOLS_TEMPLATE.format( + dsml=_DSML, + think_start=_THINK_START, + think_end=_THINK_END, + tool_schemas="\n".join(_json(dict(_function(tool))) for tool in tools), + ) + + +def _render_tool_call(tool_call: Mapping[str, Any]) -> str: + function = _function(tool_call) + raw_arguments = function.get("arguments", {}) + if isinstance(raw_arguments, str): + try: + arguments = json.loads(raw_arguments) + except (json.JSONDecodeError, TypeError): + arguments = {"arguments": raw_arguments} + elif isinstance(raw_arguments, Mapping): + # Renderer inputs allow decoded dictionaries even though the OpenAI + # wire format normally carries a JSON string. + arguments = dict(raw_arguments) + else: + arguments = {"arguments": raw_arguments} + + parameters = [] + for key, value in arguments.items(): + is_string = isinstance(value, str) + encoded = value if is_string else _json(value) + parameters.append( + f'<{_DSML}parameter name="{key}" ' + f'string="{str(is_string).lower()}">{encoded}' + f"" + ) + return ( + f'<{_DSML}invoke name="{function.get("name")}">\n' + + "\n".join(parameters) + + f"\n" + ) + + +def _merge_tool_messages(messages: list[dict[str, Any]]) -> list[dict[str, Any]]: + merged: list[dict[str, Any]] = [] + for original in messages: + message = copy.deepcopy(original) + role = message.get("role") + if role == "tool": + block = { + "type": "tool_result", + "tool_use_id": message.get("tool_call_id", ""), + "content": message.get("content", ""), + } + if ( + merged + and merged[-1].get("role") == "user" + and "content_blocks" in merged[-1] + ): + merged[-1]["content_blocks"].append(block) + else: + merged.append({"role": "user", "content_blocks": [block]}) + elif role == "user": + block = {"type": "text", "text": _text(message.get("content"))} + if ( + merged + and merged[-1].get("role") == "user" + and "content_blocks" in merged[-1] + and merged[-1].get("task") is None + ): + merged[-1]["content_blocks"].append(block) + else: + message["content"] = block["text"] + message["content_blocks"] = [block] + merged.append(message) + else: + merged.append(message) + + call_order: dict[str, int] = {} + for message in merged: + if message.get("role") == "assistant" and message.get("tool_calls"): + call_order = {} + for index, tool_call in enumerate(message["tool_calls"]): + function = _function(tool_call) + call_id = tool_call.get("id") or function.get("id") + if call_id: + call_order[str(call_id)] = index + elif message.get("role") == "user" and message.get("content_blocks"): + blocks = message["content_blocks"] + results = [b for b in blocks if b.get("type") == "tool_result"] + if len(results) > 1 and call_order: + results.sort( + key=lambda block: call_order.get(block.get("tool_use_id", ""), 0) + ) + ordered = iter(results) + message["content_blocks"] = [ + next(ordered) if block.get("type") == "tool_result" else block + for block in blocks + ] + return merged + + +def _drop_historical_thinking( + messages: list[dict[str, Any]], +) -> list[dict[str, Any]]: + last_user = max( + ( + index + for index, message in enumerate(messages) + if message.get("role") in {"user", "developer"} + ), + default=-1, + ) + kept = [] + for index, original in enumerate(messages): + role = original.get("role") + if role in {"user", "system", "tool", "latest_reminder"} or index >= last_user: + kept.append(original) + elif role == "assistant": + message = copy.copy(original) + message.pop("reasoning_content", None) + kept.append(message) + return kept + + +def _render_deepseek_v4_reference( + messages: list[dict[str, Any]], + *, + tools: list[dict[str, Any]] | None, + add_generation_prompt: bool, + enable_thinking: bool, + drop_thinking: bool, + reasoning_effort: str, +) -> str: + messages = copy.deepcopy(messages) + + if tools: + target: dict[str, Any] | None = next( + (m for m in messages if m.get("role") == "developer"), + None, + ) + if target is None: + target = next( + (m for m in messages if m.get("role") == "system"), + None, + ) + if target is None: + target = {"role": "system", "content": ""} + messages.insert(0, target) + target["tools"] = copy.deepcopy(tools) + + messages = _merge_tool_messages(messages) + effective_drop = drop_thinking and not any(m.get("tools") for m in messages) + if enable_thinking and effective_drop: + messages = _drop_historical_thinking(messages) + + last_user = max( + ( + index + for index, message in enumerate(messages) + if message.get("role") in {"user", "developer"} + ), + default=-1, + ) + prompt = _BOS + if enable_thinking: + prompt += _REASONING_EFFORT_PROMPTS[reasoning_effort] + + for index, message in enumerate(messages): + role = message.get("role") + content = message.get("content") + + if role == "system": + prompt += _text(content) + if message.get("tools"): + prompt += "\n\n" + _render_tools(message["tools"]) + if message.get("response_format") is not None: + prompt += ( + "\n\n## Response Format:\n\n" + "You MUST strictly adhere to the following schema to reply:\n" + + _json(message["response_format"]) + ) + elif role == "developer": + prompt += _USER + _text(content) + if message.get("tools"): + prompt += "\n\n" + _render_tools(message["tools"]) + if message.get("response_format") is not None: + prompt += ( + "\n\n## Response Format:\n\n" + "You MUST strictly adhere to the following schema to reply:\n" + + _json(message["response_format"]) + ) + elif role == "user": + prompt += _USER + rendered_blocks = [] + for block in message.get("content_blocks") or []: + if block.get("type") == "text": + rendered_blocks.append(_text(block.get("text"))) + elif block.get("type") == "tool_result": + rendered_blocks.append( + f"{_text(block.get('content'))}" + ) + prompt += "\n\n".join(rendered_blocks) + elif role == "latest_reminder": + prompt += _LATEST_REMINDER + _text(content) + elif role == "assistant": + previous_has_task = ( + index > 0 and messages[index - 1].get("task") is not None + ) + keep_reasoning = ( + enable_thinking + and not previous_has_task + and (not effective_drop or index > last_user) + ) + if keep_reasoning: + prompt += str(message.get("reasoning_content") or "") + _THINK_END + prompt += _text(content) + tool_calls = message.get("tool_calls") or [] + if tool_calls: + prompt += ( + f"\n\n<{_DSML}tool_calls>\n" + + "\n".join(_render_tool_call(call) for call in tool_calls) + + f"\n" + ) + if not message.get("wo_eos", False): + prompt += _EOS + else: + raise ValueError(f"Unsupported DeepSeek V4 reference role: {role!r}") + + next_role = ( + messages[index + 1].get("role") if index + 1 < len(messages) else None + ) + if next_role is not None and next_role not in { + "assistant", + "latest_reminder", + }: + continue + + task = message.get("task") + if task is not None: + if task not in _TASK_TOKENS: + raise ValueError(f"Invalid DeepSeek V4 reference task: {task!r}") + if task != "action": + prompt += _TASK_TOKENS[task] + else: + prompt += _ASSISTANT + prompt += _THINK_START if enable_thinking else _THINK_END + prompt += _TASK_TOKENS[task] + continue + + if (next_role is None and not add_generation_prompt) or role not in { + "user", + "developer", + }: + continue + prompt += _ASSISTANT + if enable_thinking and (not effective_drop or index >= last_user): + prompt += _THINK_START + else: + prompt += _THINK_END + + return prompt + + +def render_reference(tokenizer, messages: list[dict[str, Any]], **kwargs) -> list[int]: + """Render ``messages`` through the model's independent reference oracle.""" + kwargs = dict(kwargs) + kwargs.setdefault("add_generation_prompt", False) + model_name = getattr(tokenizer, "name_or_path", "") + + if model_name == DEEPSEEK_V4_MODEL: + text = _render_deepseek_v4_reference( + messages, + tools=kwargs.pop("tools", None), + add_generation_prompt=kwargs.pop("add_generation_prompt"), + enable_thinking=kwargs.pop("enable_thinking", False), + drop_thinking=kwargs.pop("drop_thinking", True), + reasoning_effort=kwargs.pop("reasoning_effort", "low"), + ) + if kwargs: + raise TypeError( + f"Unsupported DeepSeek V4 reference kwargs: {sorted(kwargs)}" + ) + return list(tokenizer.encode(text, add_special_tokens=False)) + + result = tokenizer.apply_chat_template( + messages, + tokenize=True, + return_dict=False, + **kwargs, + ) + if isinstance(result, dict): + return list(result["input_ids"]) + if isinstance(result, str): + return list(tokenizer.encode(result, add_special_tokens=False)) + return list(result) + + +__all__ = ["DEEPSEEK_V4_MODEL", "render_reference"] diff --git a/tests/test_bridge.py b/tests/test_bridge.py index 1f36bbfd..93b049a7 100644 --- a/tests/test_bridge.py +++ b/tests/test_bridge.py @@ -34,6 +34,7 @@ ("zai-org/GLM-5", "auto"), ("zai-org/GLM-5.1", "auto"), ("THUDM/GLM-4.5-Air", "auto"), + ("deepseek-ai/DeepSeek-V4-Flash-0731", "auto"), ("MiniMaxAI/MiniMax-M2.5", "auto"), ("moonshotai/Kimi-K2-Instruct", "auto"), ("moonshotai/Kimi-K2.5", "auto"), diff --git a/tests/test_build_helpers.py b/tests/test_build_helpers.py index 5186ddce..110927ad 100644 --- a/tests/test_build_helpers.py +++ b/tests/test_build_helpers.py @@ -7,6 +7,7 @@ from renderers import build_training_sample, build_trajectory_step from renderers.base import PlaceholderRange, _build_mm_token_type_ids +from tests.reference_rendering import render_reference def test_build_mm_token_type_ids_marks_ranges(): @@ -20,23 +21,11 @@ def test_build_mm_token_type_ids_marks_ranges(): def _expected(tokenizer, messages, **kwargs): - # Match the Renderer Protocol's default for add_generation_prompt - # (False); some tokenizers default it to True in their config - # (e.g. Kimi) which would otherwise flip the parity check on the flag - # alone. Callers wanting the gen prompt still pass it through. - kwargs.setdefault("add_generation_prompt", False) - result = tokenizer.apply_chat_template( - messages, tokenize=True, return_dict=False, **kwargs - ) - if isinstance(result, dict): - return list(result["input_ids"]) - if isinstance(result, str): - return list(tokenizer.encode(result, add_special_tokens=False)) - return list(result) + return render_reference(tokenizer, messages, **kwargs) def test_build_training_sample_ids_match(model_name, tokenizer, renderer): - """Token IDs must match apply_chat_template.""" + """Token IDs must match the model-aware reference renderer.""" if ( model_name in { diff --git a/tests/test_deepseek_v4.py b/tests/test_deepseek_v4.py new file mode 100644 index 00000000..43ab8f35 --- /dev/null +++ b/tests/test_deepseek_v4.py @@ -0,0 +1,565 @@ +"""DeepSeek V4 Flash 0731 reference-encoder and DSML coverage.""" + +from __future__ import annotations + +from functools import lru_cache + +import pytest +from pydantic import TypeAdapter, ValidationError + +from renderers import ( + DeepSeekV4Renderer, + DeepSeekV4RendererConfig, + RendererConfig, + ToolCallParseStatus, + create_renderer, +) +from renderers.base import MODEL_RENDERER_MAP, load_tokenizer +from tests.reference_rendering import render_reference + + +MODEL = "deepseek-ai/DeepSeek-V4-Flash-0731" +BOS = "<｜begin▁of▁sentence｜>" +EOS = "<｜end▁of▁sentence｜>" +USER = "<｜User｜>" +ASSISTANT = "<｜Assistant｜>" + +TOOLS = [ + { + "type": "function", + "function": { + "name": "weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": { + "city": {"type": "string"}, + "days": {"type": "integer"}, + }, + "required": ["city"], + }, + }, + } +] + + +@lru_cache(maxsize=1) +def _tokenizer(): + return load_tokenizer(MODEL) + + +def _renderer(**config_kwargs): + return DeepSeekV4Renderer( + _tokenizer(), + DeepSeekV4RendererConfig(**config_kwargs), + ) + + +def _decode(renderer, messages, **kwargs): + return _tokenizer().decode( + renderer.render_ids(messages, **kwargs), + skip_special_tokens=False, + ) + + +def test_registration_and_native_defaults(): + tokenizer = _tokenizer() + renderer = create_renderer(tokenizer) + + assert tokenizer.chat_template is None + assert MODEL_RENDERER_MAP[MODEL] == "deepseek-v4" + assert isinstance(renderer, DeepSeekV4Renderer) + assert renderer.config.enable_thinking is False + assert renderer.config.drop_thinking is True + assert renderer.config.reasoning_effort == "low" + assert renderer.effective_thinking_retention == "all" + + +def test_config_discriminator_and_template_kwarg_contract(): + parsed = TypeAdapter(RendererConfig).validate_python( + { + "name": "deepseek-v4", + "enable_thinking": True, + "drop_thinking": False, + "reasoning_effort": "max", + } + ) + assert isinstance(parsed, DeepSeekV4RendererConfig) + assert parsed.reasoning_effort == "max" + + with pytest.raises(ValidationError): + DeepSeekV4RendererConfig( + drop_thinking=False, + thinking_retention="tool_cycle", + ) + + +def test_chat_mode_generation_prompt_matches_reference_encoder(): + messages = [ + {"role": "system", "content": "Be concise."}, + {"role": "user", "content": "Hello"}, + ] + + assert _decode(_renderer(), messages, add_generation_prompt=True) == ( + f"{BOS}Be concise.{USER}Hello{ASSISTANT}" + ) + assert _decode(_renderer(), messages) == f"{BOS}Be concise.{USER}Hello" + + +def test_thinking_mode_drops_only_historical_reasoning_without_tools(): + messages = [ + {"role": "system", "content": "Be concise."}, + {"role": "user", "content": "First"}, + { + "role": "assistant", + "reasoning_content": "old secret", + "content": "First answer", + }, + {"role": "user", "content": "Second"}, + { + "role": "assistant", + "reasoning_content": "current thought", + "content": "Second answer", + }, + ] + + assert _decode(_renderer(enable_thinking=True), messages) == ( + f"{BOS}Be concise." + f"{USER}First{ASSISTANT}First answer{EOS}" + f"{USER}Second{ASSISTANT}current thoughtSecond answer{EOS}" + ) + + +def test_tools_preserve_reasoning_and_use_dsml_wire_format(): + messages = [ + {"role": "system", "content": "Be helpful."}, + {"role": "user", "content": "Weather?"}, + { + "role": "assistant", + "reasoning_content": "I should call the tool.", + "content": "", + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": { + "name": "weather", + "arguments": {"city": "Berlin", "days": 2}, + }, + } + ], + }, + {"role": "tool", "tool_call_id": "call-1", "content": '{"sun":true}'}, + ] + + text = _decode( + _renderer(enable_thinking=True), + messages, + tools=TOOLS, + add_generation_prompt=True, + ) + + assert text.startswith(f"{BOS}Be helpful.\n\n## Tools\n") + assert f"{USER}Weather?{ASSISTANT}I should call the tool." in text + assert '<｜DSML｜invoke name="weather">' in text + assert ( + '<｜DSML｜parameter name="city" string="true">Berlin' + ) in text + assert ( + '<｜DSML｜parameter name="days" string="false">2' + ) in text + assert ( + f'{EOS}{USER}{{"sun":true}}{ASSISTANT}' + ) in text + + +def test_parallel_tool_results_are_sorted_by_call_order(): + messages = [ + {"role": "user", "content": "Run both"}, + { + "role": "assistant", + "tool_calls": [ + {"id": "a", "function": {"name": "first", "arguments": {}}}, + {"id": "b", "function": {"name": "second", "arguments": {}}}, + ], + }, + {"role": "tool", "tool_call_id": "b", "content": "second result"}, + {"role": "tool", "tool_call_id": "a", "content": "first result"}, + ] + + text = _decode(_renderer(), messages) + assert text.index("first result") < text.index("second result") + + +def test_dsml_roundtrip_preserves_string_and_json_argument_types(): + renderer = _renderer(enable_thinking=True) + messages = [ + {"role": "user", "content": "Weather?"}, + { + "role": "assistant", + "reasoning_content": "Use weather.", + "content": "checking", + "tool_calls": [ + { + "function": { + "name": "weather", + "arguments": { + "city": "true", + "days": 2, + "flags": [True, False], + }, + } + } + ], + }, + ] + rendered = renderer.render_ids(messages) + assistant_id = _tokenizer().encode(ASSISTANT, add_special_tokens=False)[0] + completion_start = rendered.index(assistant_id) + 2 # skip Assistant + + + parsed = renderer.parse_response(rendered[completion_start:]) + + assert parsed.reasoning_content == "Use weather." + assert parsed.content == "checking" + assert len(parsed.tool_calls) == 1 + call = parsed.tool_calls[0] + assert call.status == ToolCallParseStatus.OK + assert call.name == "weather" + assert call.arguments == { + "city": "true", + "days": 2, + "flags": [True, False], + } + assert call.token_span is not None + + +@pytest.mark.parametrize( + "arguments, decoded_type", + [ + ("[]", "list"), + ('"value"', "str"), + ("1", "int"), + ("null", "NoneType"), + ], +) +def test_json_nonobject_tool_arguments_raise_like_reference_encoder( + arguments, + decoded_type, +): + messages = [ + {"role": "user", "content": "Call it"}, + { + "role": "assistant", + "tool_calls": [ + { + "function": { + "name": "weather", + "arguments": arguments, + } + } + ], + }, + ] + + with pytest.raises(AttributeError) as reference_error: + render_reference(_tokenizer(), messages) + with pytest.raises(AttributeError) as renderer_error: + _renderer().render_ids(messages) + + expected = f"'{decoded_type}' object has no attribute 'items'" + assert str(reference_error.value) == expected + assert str(renderer_error.value) == expected + + +def test_reasoning_effort_prefix_is_after_bos_and_thinking_only(): + messages = [{"role": "user", "content": "Think"}] + high = _decode( + _renderer(enable_thinking=True, reasoning_effort="high"), + messages, + add_generation_prompt=True, + ) + chat = _decode( + _renderer(enable_thinking=False, reasoning_effort="high"), + messages, + add_generation_prompt=True, + ) + + assert high.startswith(f"{BOS}Reasoning Effort: Absolute maximum") + assert chat == f"{BOS}{USER}Think{ASSISTANT}" + + +def test_reference_encoder_drops_stale_developer_messages_without_tools(): + messages = [ + {"role": "developer", "content": "stale internal query"}, + {"role": "assistant", "reasoning_content": "old", "content": "old answer"}, + {"role": "user", "content": "current public query"}, + ] + + text = _decode( + _renderer(enable_thinking=True), + messages, + add_generation_prompt=True, + ) + + assert "stale internal query" not in text + assert text == (f"{BOS}old answer{EOS}{USER}current public query{ASSISTANT}") + + +def test_quick_task_token_renders_without_normal_generation_prompt(): + messages = [{"role": "user", "content": "Search?", "task": "action"}] + + renderer = _renderer() + assert renderer.render_ids(messages) == render_reference( + _tokenizer(), + messages, + ) + assert _decode(renderer, messages) == ( + f"{BOS}{USER}Search?{ASSISTANT}<｜action｜>" + ) + + +def test_action_task_assistant_suppresses_thinking_like_reference_encoder(): + messages = [ + {"role": "user", "content": "Classify", "task": "action"}, + { + "role": "assistant", + "reasoning_content": "must not render", + "content": "result", + }, + ] + renderer = _renderer(enable_thinking=True) + + rendered = renderer.render_ids(messages) + + assert rendered == render_reference( + _tokenizer(), + messages, + enable_thinking=True, + ) + assert _tokenizer().decode(rendered, skip_special_tokens=False) == ( + f"{BOS}{USER}Classify{ASSISTANT}<｜action｜>result{EOS}" + ) + + +def test_historical_action_task_keeps_reference_encoders_unclosed_think(): + messages = [ + {"role": "user", "content": "Classify", "task": "action"}, + {"role": "assistant", "content": "result"}, + {"role": "user", "content": "Continue"}, + ] + renderer = _renderer(enable_thinking=True) + + rendered = renderer.render_ids(messages, add_generation_prompt=True) + + assert rendered == render_reference( + _tokenizer(), + messages, + add_generation_prompt=True, + enable_thinking=True, + ) + assert _tokenizer().decode(rendered, skip_special_tokens=False) == ( + f"{BOS}{USER}Classify{ASSISTANT}<｜action｜>result{EOS}" + f"{USER}Continue{ASSISTANT}" + ) + + +def test_task_before_nonassistant_does_not_emit_task_token(): + messages = [ + {"role": "user", "content": "First", "task": "query"}, + {"role": "user", "content": "Second"}, + ] + renderer = _renderer(enable_thinking=True) + + rendered = renderer.render_ids(messages, add_generation_prompt=True) + + assert rendered == render_reference( + _tokenizer(), + messages, + add_generation_prompt=True, + enable_thinking=True, + ) + assert "<｜query｜>" not in _tokenizer().decode( + rendered, + skip_special_tokens=False, + ) + + +def test_tool_result_merges_into_tasked_user_like_reference_encoder(): + messages = [ + {"role": "user", "content": "Classify", "task": "action"}, + {"role": "tool", "tool_call_id": "call-1", "content": "done"}, + { + "role": "assistant", + "reasoning_content": "must not render", + "content": "result", + }, + ] + renderer = _renderer(enable_thinking=True) + + rendered = renderer.render_ids(messages) + + assert rendered == render_reference( + _tokenizer(), + messages, + enable_thinking=True, + ) + assert _tokenizer().decode(rendered, skip_special_tokens=False) == ( + f"{BOS}{USER}Classify\n\ndone" + f"{ASSISTANT}<｜action｜>result{EOS}" + ) + + +def test_merged_followup_task_is_dropped_like_reference_encoder(): + messages = [ + {"role": "tool", "tool_call_id": "call-1", "content": "done"}, + {"role": "user", "content": "Classify", "task": "action"}, + ] + renderer = _renderer(enable_thinking=True) + + rendered = renderer.render_ids(messages, add_generation_prompt=True) + + assert rendered == render_reference( + _tokenizer(), + messages, + add_generation_prompt=True, + enable_thinking=True, + ) + text = _tokenizer().decode(rendered, skip_special_tokens=False) + assert "<｜action｜>" not in text + assert text.endswith(f"{ASSISTANT}") + + +@pytest.mark.parametrize("prefix", ["", "\n"]) +def test_dsml_parser_requires_reference_encoders_two_newlines(prefix): + text = ( + f"{prefix}<｜DSML｜tool_calls>\n" + '<｜DSML｜invoke name="weather">\n\n\n' + "" + ) + + parsed = _renderer().parse_response( + _tokenizer().encode(text, add_special_tokens=False) + ) + + assert parsed.content == text + assert parsed.tool_calls == [] + + +def test_rendered_masks_keep_dsml_sampled_and_tool_wrappers_scaffolded(): + renderer = _renderer(enable_thinking=True) + messages = [ + {"role": "user", "content": "Call it"}, + { + "role": "assistant", + "reasoning_content": "calling", + "tool_calls": [ + {"id": "x", "function": {"name": "weather", "arguments": {}}} + ], + }, + {"role": "tool", "tool_call_id": "x", "content": "sunny"}, + ] + rendered = renderer.render(messages, tools=TOOLS) + + assert len(rendered.token_ids) == len(rendered.message_indices) + assert len(rendered.token_ids) == len(rendered.sampled_mask) + assert len(rendered.token_ids) == len(rendered.is_content) + assert rendered.tokens_by_role(sampled_only=True)["assistant"] > 0 + assert rendered.tokens_by_role(sampled_only=True)["tool"] == 0 + tool_content = rendered.content_mask_for_roles({"tool"}) + assert ( + _tokenizer().decode( + [token for token, keep in zip(rendered.token_ids, tool_content) if keep], + skip_special_tokens=False, + ) + == "sunny" + ) + + +def test_bridge_extends_a_single_tool_result_exactly(): + renderer = _renderer(enable_thinking=True) + first_messages = [ + {"role": "user", "content": "Call it"}, + { + "role": "assistant", + "reasoning_content": "calling", + "tool_calls": [ + { + "id": "x", + "function": {"name": "weather", "arguments": {"city": "Rome"}}, + } + ], + }, + ] + full_messages = first_messages + [ + {"role": "tool", "tool_call_id": "x", "content": "sunny"} + ] + prompt = renderer.render_ids( + first_messages[:1], + tools=TOOLS, + add_generation_prompt=True, + ) + full_first = renderer.render_ids(first_messages, tools=TOOLS) + completion = full_first[len(prompt) :] + + bridged = renderer.bridge_to_next_turn( + prompt, + completion, + full_messages[-1:], + tools=TOOLS, + ) + + assert bridged is not None + assert bridged.token_ids == renderer.render_ids( + full_messages, + tools=TOOLS, + add_generation_prompt=True, + ) + + +def test_bridge_declines_at_developer_query_boundary_when_dropping_thinking(): + renderer = _renderer(enable_thinking=True) + prior_messages = [{"role": "user", "content": "Q1"}] + answer = { + "role": "assistant", + "reasoning_content": "old reasoning", + "content": "A1", + } + new_messages = [{"role": "developer", "content": "Q2"}] + prompt = renderer.render_ids(prior_messages, add_generation_prompt=True) + completed = renderer.render_ids([*prior_messages, answer]) + completion = completed[len(prompt) :] + + assert renderer.bridge_to_next_turn(prompt, completion, new_messages) is None + + full_messages = [*prior_messages, answer, *new_messages] + assert renderer.render_ids( + full_messages, + add_generation_prompt=True, + ) == render_reference( + _tokenizer(), + full_messages, + enable_thinking=True, + add_generation_prompt=True, + ) + + +def test_bridge_extends_developer_query_when_preserving_all_thinking(): + renderer = _renderer(enable_thinking=True, drop_thinking=False) + prior_messages = [{"role": "user", "content": "Q1"}] + answer = { + "role": "assistant", + "reasoning_content": "retained reasoning", + "content": "A1", + } + new_messages = [{"role": "developer", "content": "Q2"}] + prompt = renderer.render_ids(prior_messages, add_generation_prompt=True) + completed = renderer.render_ids([*prior_messages, answer]) + completion = completed[len(prompt) :] + + bridged = renderer.bridge_to_next_turn(prompt, completion, new_messages) + + assert bridged is not None + assert bridged.token_ids == renderer.render_ids( + [*prior_messages, answer, *new_messages], + add_generation_prompt=True, + ) diff --git a/tests/test_render_ids.py b/tests/test_render_ids.py index 52628e14..d9d4c08b 100644 --- a/tests/test_render_ids.py +++ b/tests/test_render_ids.py @@ -1,8 +1,11 @@ -"""Barrage test: renderer.render_ids() must match tokenizer.apply_chat_template(). +"""Barrage test: renderer.render_ids() must match its reference renderer. Every test case runs against every (model, renderer) pair from conftest. If a test passes, the renderer is token-for-token correct for that case. +The shared reference helper uses Hugging Face ``apply_chat_template`` for Jinja +models and the official Python encoding contract for DeepSeek V4. + GPT-OSS is auto-skipped here by ``conftest._skip_gpt_oss_for_hf_parity_tests`` since our GptOssRenderer matches openai-harmony / vLLM, not the HF Jinja template. See ``test_gpt_oss_harmony_parity.py`` for harmony parity coverage. @@ -14,6 +17,7 @@ from renderers import create_renderer from renderers.base import load_tokenizer +from tests.reference_rendering import render_reference _GEMMA4_EMPTY_THOUGHT_MODELS = { "google/gemma-4-26B-A4B-it", @@ -35,20 +39,7 @@ def _expected(tokenizer, messages, **kwargs): "Gemma 4 26B/31B deliberately re-emits the disabled-thinking " "generation prefill on assistant history for sampled-token stability" ) - # Match the Renderer Protocol's default for add_generation_prompt (False) - # — some tokenizers (e.g. Kimi's) default it to True in their config, - # which would otherwise make this parity check fail on the flag alone. - # Callers that explicitly want the gen prompt still pass it through. - kwargs.setdefault("add_generation_prompt", False) - result = tokenizer.apply_chat_template( - messages, tokenize=True, return_dict=False, **kwargs - ) - if isinstance(result, dict): - return list(result["input_ids"]) - if isinstance(result, str): - # Some tokenizers return str even with tokenize=True; force encode - return list(tokenizer.encode(result, add_special_tokens=False)) - return list(result) + return render_reference(tokenizer, messages, **kwargs) # ── Basic messages ─────────────────────────────────────────────────── diff --git a/tests/test_renderer_config_parity.py b/tests/test_renderer_config_parity.py index 745b2cf8..56e1e192 100644 --- a/tests/test_renderer_config_parity.py +++ b/tests/test_renderer_config_parity.py @@ -1,13 +1,11 @@ -"""Parity for typed-config template fields against the upstream chat -template. +"""Parity for typed-config template fields against the upstream reference. Each renderer's typed config (see ``renderers.configs``) declares the fields that mirror chat-template kwargs via ``Config.template_field_names()``. ``test_renderer_config.py`` covers the typed-config wiring; this file covers the only thing that matters downstream: that flipping a template field on the typed config produces -token streams byte-identical to -``tokenizer.apply_chat_template(messages, **{field: value})``. +token streams byte-identical to the model-aware reference renderer. Without this, the typed surface is a promise the renderer doesn't keep. @@ -37,6 +35,7 @@ load_tokenizer, ) from renderers.configs import _config_class_for +from tests.reference_rendering import render_reference # Models exercised by the parity tests. Mirrors ``conftest.RENDERER_MODELS`` @@ -58,6 +57,7 @@ ("moonshotai/Kimi-K2.6", "auto"), ("deepseek-ai/DeepSeek-V3", "auto"), ("deepseek-ai/DeepSeek-R1", "auto"), + ("deepseek-ai/DeepSeek-V4-Flash-0731", "auto"), # Nano + Super share the ``nemotron-3`` config (incl. ``low_effort``, which # fires only on Super); both are exercised so the kwarg is checked where it # no-ops (Nano) AND where it appends (Super). @@ -95,7 +95,10 @@ # gpt-oss accepts low/medium/high; Hy3 accepts no_think/low/high. The # union is listed here and the matrix builder drops values a given # renderer's typed config rejects (see ``_value_valid_for``). - "reasoning_effort": ["no_think", "low", "medium", "high", "xhigh"], + "reasoning_effort": ["no_think", "low", "medium", "high", "xhigh", "max"], + # DeepSeek V4 — evict reasoning before the latest user query when tools + # are absent. Tools force preservation in the official Python encoder. + "drop_thinking": [True, False], # Hy3 — keep {reasoning} on historical assistant turns # (True) vs collapse past-cycle reasoning to (False). "preserved_thinking": [True, False], @@ -304,9 +307,9 @@ def _value_valid_for(model: str, renderer_name: str, kwarg: str, value: Any) -> return False -def _hf_parity_matrix() -> list[Any]: +def _reference_parity_matrix() -> list[Any]: """Auto-derived ``(model, renderer_name, kwarg, value)`` matrix for - every renderer with template fields, minus gpt-oss (handled + every renderer with reference-controlled fields, minus gpt-oss (handled separately against harmony). """ out = [] @@ -376,9 +379,10 @@ def _renderer_with_kwarg(model_name: str, renderer_name: str, kwarg: str, value: return create_renderer(tok, config) -def _expected_hf(tokenizer, messages, *, kwarg: str, value: Any, **render_kwargs): - """Render via ``apply_chat_template`` with the kwarg spread as a - top-level argument. +def _expected_reference( + tokenizer, messages, *, kwarg: str, value: Any, **render_kwargs +): + """Render through the model-aware oracle with one config field changed. transformers v5.x silently drops ``chat_template_kwargs={...}`` — only direct kwargs propagate into the Jinja environment. The two @@ -388,31 +392,24 @@ def _expected_hf(tokenizer, messages, *, kwarg: str, value: Any, **render_kwargs in OpenAI-compatible servers; we translate it to constructor kwargs on our side.) """ - render_kwargs.setdefault("add_generation_prompt", False) - result = tokenizer.apply_chat_template( + return render_reference( + tokenizer, messages, - tokenize=True, - return_dict=False, **{kwarg: value}, **render_kwargs, ) - if isinstance(result, dict): - return list(result["input_ids"]) - if isinstance(result, str): - return list(tokenizer.encode(result, add_special_tokens=False)) - return list(result) -# ── HF-Jinja parity (every renderer except gpt-oss) ──────────────────── +# ── Reference parity (every renderer except gpt-oss) ────────────────── -@pytest.mark.parametrize("model,renderer_name,kwarg,value", _hf_parity_matrix()) +@pytest.mark.parametrize("model,renderer_name,kwarg,value", _reference_parity_matrix()) @pytest.mark.parametrize( "shape_id,messages,render_kwargs", _MESSAGE_SHAPES, ids=[s[0] for s in _MESSAGE_SHAPES], ) -def test_chat_template_kwarg_parity_hf( +def test_chat_template_kwarg_parity_reference( model, renderer_name, kwarg, @@ -462,19 +459,18 @@ def test_chat_template_kwarg_parity_hf( ) try: - expected = _expected_hf( + expected = _expected_reference( tokenizer, messages, kwarg=kwarg, value=value, **render_kwargs ) except Exception as exc: pytest.xfail( - f"{model}: apply_chat_template raised {type(exc).__name__}: " - f"{str(exc)[:160]}" + f"{model}: reference renderer raised {type(exc).__name__}: {str(exc)[:160]}" ) got = renderer.render_ids(messages, **render_kwargs) assert got == expected, ( f"{model} / shape={shape_id} / {kwarg}={value}: renderer diverged " - f"from apply_chat_template (len got={len(got)}, expected={len(expected)})" + f"from its reference renderer (len got={len(got)}, expected={len(expected)})" ) diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index ab36bf7a..4a0760b6 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -42,6 +42,7 @@ ("zai-org/GLM-5.1", "auto"), ("zai-org/GLM-4.7-Flash", "auto"), ("THUDM/GLM-4.5-Air", "auto"), + ("deepseek-ai/DeepSeek-V4-Flash-0731", "auto"), ("MiniMaxAI/MiniMax-M2.5", "auto"), ("moonshotai/Kimi-K2-Instruct", "auto"), ("moonshotai/Kimi-K2.5", "auto"),