diff --git a/.github/workflows/copyright-check.ps1 b/.github/workflows/copyright-check.ps1 index 5613d51ff63f..9411e21183f8 100644 --- a/.github/workflows/copyright-check.ps1 +++ b/.github/workflows/copyright-check.ps1 @@ -84,7 +84,7 @@ $global:copyright_results = @{ $ignored_files = @('.clang-format', '.gitattributes', '.gitignore', '.gitkeep', '.patch', 'Cargo.lock', 'LICENSE', 'uv.lock', 'rust-toolchain.toml', 'codespell.txt', 'exclusions.txt') write-debug " ignored_files = ['$($ignored_files -join "','")']." -$ignored_paths = @('.github', '.mypy_cache', '.pytest_cache', 'lib/llm/tests/data/sample-models', 'lib/llm/tests/data/deepseek-v3.2') +$ignored_paths = @('.github', '.mypy_cache', '.pytest_cache', 'lib/llm/tests/data/sample-models', 'lib/llm/tests/data/deepseek-v3.2', 'lib/llm/tests/data/deepseek-v4') write-debug " ignored_paths = ['$($ignored_paths -join "','")']." $ignored_types = @('.bat', '.gif', '.ico', '.ipynb', '.jpg', '.jpeg', '.patch', '.png', '.pyc', '.pyi', '.rst', '.zip', '.md', '.json') write-debug " ignored_types = ['$($ignored_types -join "', '")']." diff --git a/components/src/dynamo/frontend/sglang_prepost.py b/components/src/dynamo/frontend/sglang_prepost.py index c66ee450760f..ec9ebf0f6830 100644 --- a/components/src/dynamo/frontend/sglang_prepost.py +++ b/components/src/dynamo/frontend/sglang_prepost.py @@ -3,9 +3,12 @@ from __future__ import annotations +import copy +import inspect import json import logging from dataclasses import dataclass +from functools import lru_cache from typing import Any, TypeAlias from sglang.srt.entrypoints.openai.protocol import Function as SglangFunction @@ -129,6 +132,127 @@ def _is_named_tool_choice(tool_choice: Any) -> bool: ) +def _normalize_deepseek_v4_hint(value: Any) -> str: + return str(value or "").lower().replace("-", "").replace("_", "") + + +def _should_use_deepseek_v4_encoding( + request: dict[str, Any], + *, + tokenizer, + tool_call_parser_name: str | None, + reasoning_parser_name: str | None, +) -> bool: + if getattr(tokenizer, "chat_template", None) is not None: + return False + + return any( + "deepseekv4" in _normalize_deepseek_v4_hint(value) + for value in ( + request.get("model"), + tool_call_parser_name, + reasoning_parser_name, + ) + ) + + +def _filter_template_tools( + request: dict[str, Any], + *, + exclude_tools_when_tool_choice_none: bool, +) -> list[dict[str, Any]] | None: + raw_tools = request.get("tools") or [] + if not raw_tools: + return None + + tool_choice = request.get("tool_choice", "auto") + if exclude_tools_when_tool_choice_none and tool_choice == "none": + return None + + if _is_named_tool_choice(tool_choice): + chosen_name = tool_choice["function"]["name"] + return [ + copy.deepcopy(tool) + for tool in raw_tools + if tool.get("function", {}).get("name") == chosen_name + ] + + return copy.deepcopy(raw_tools) + + +def _render_deepseek_v4_prompt_token_ids( + request: dict[str, Any], + *, + messages: list[dict[str, Any]], + tokenizer, + template_tools: list[dict[str, Any]] | None, +) -> list[int]: + try: + from sglang.srt.entrypoints.openai.encoding_dsv4 import encode_messages + except ImportError as exc: + raise ValueError( + "DeepSeek-V4 preprocessing requires SGLang's " + "sglang.srt.entrypoints.openai.encoding_dsv4 encoder. " + "Install an SGLang build that includes the DeepSeek-V4 integration." + ) from exc + + encoding_messages = copy.deepcopy(messages) + for msg in encoding_messages: + if msg.get("content") is None: + msg["content"] = "" + + if template_tools: + if not encoding_messages or encoding_messages[0].get("role") != "system": + encoding_messages.insert(0, {"role": "system", "content": ""}) + encoding_messages[0]["tools"] = template_tools + + chat_template_kwargs = request.get("chat_template_kwargs") or {} + thinking_mode = "thinking" if chat_template_kwargs.get("thinking") else "chat" + reasoning_effort = ( + request.get("reasoning_effort") + or chat_template_kwargs.get("reasoning_effort") + or None + ) + if reasoning_effort not in ("max", "high", None): + reasoning_effort = None + + prompt = encode_messages( + encoding_messages, + thinking_mode=thinking_mode, + reasoning_effort=reasoning_effort, + ) + return _normalize_prompt_token_ids(tokenizer.encode(prompt)) + + +@lru_cache(maxsize=64) +def _callable_accepts_kwarg(func: Any, kwarg: str) -> bool: + try: + signature = inspect.signature(func) + except (TypeError, ValueError): + return False + + for name, param in signature.parameters.items(): + if param.kind == inspect.Parameter.VAR_KEYWORD: + return True + if name == kwarg and param.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ): + return True + return False + + +def _call_with_optional_parallel_tool_calls( + func: Any, + *args: Any, + parallel_tool_calls: Any, +) -> Any: + """Call SGLang helpers across versions with/without parallel_tool_calls.""" + if _callable_accepts_kwarg(func, "parallel_tool_calls"): + return func(*args, parallel_tool_calls=parallel_tool_calls) + return func(*args) + + def build_tool_call_guided_decoding( request: dict[str, Any], *, @@ -161,7 +285,8 @@ def build_tool_call_guided_decoding( ) constraint = ( "json_schema", - get_json_schema_constraint( + _call_with_optional_parallel_tool_calls( + get_json_schema_constraint, sglang_tools, sglang_tool_choice, parallel_tool_calls=parallel_tool_calls, @@ -172,7 +297,8 @@ def build_tool_call_guided_decoding( tools=sglang_tools, tool_call_parser=tool_call_parser_name, ) - constraint = parser.get_structure_constraint( + constraint = _call_with_optional_parallel_tool_calls( + parser.get_structure_constraint, tool_choice, parallel_tool_calls=parallel_tool_calls, ) @@ -239,30 +365,38 @@ def preprocess_chat_request( f"present in tools (available: {sorted(available_names) or 'none'})" ) - # Build template kwargs -- single call for rendering + tokenization - template_kwargs: dict[str, Any] = { - "add_generation_prompt": True, - "tokenize": True, - } - # Strip tools from template when tool_choice=none so the model doesn't - # see them and generate raw XML tool calls in its response. - # When tool_choice names a specific function, only include that tool - # in the template so the model doesn't see irrelevant definitions. - if sglang_tools and not ( - exclude_tools_when_tool_choice_none and tool_choice == "none" - ): - if _is_named_tool_choice(tool_choice): - chosen_name = tool_choice["function"]["name"] - template_kwargs["tools"] = [ - t.model_dump() for t in sglang_tools if t.function.name == chosen_name - ] - else: - template_kwargs["tools"] = [t.model_dump() for t in sglang_tools] - - prompt_token_ids = _normalize_prompt_token_ids( - tokenizer.apply_chat_template(messages, **template_kwargs) + template_tools = _filter_template_tools( + request, + exclude_tools_when_tool_choice_none=exclude_tools_when_tool_choice_none, ) + if _should_use_deepseek_v4_encoding( + request, + tokenizer=tokenizer, + tool_call_parser_name=tool_call_parser_name, + reasoning_parser_name=reasoning_parser_name, + ): + prompt_token_ids = _render_deepseek_v4_prompt_token_ids( + request, + messages=messages, + tokenizer=tokenizer, + template_tools=template_tools, + ) + else: + # Build template kwargs -- single call for rendering + tokenization + template_kwargs: dict[str, Any] = { + "add_generation_prompt": True, + "tokenize": True, + } + if template_tools: + template_kwargs["tools"] = template_tools + + prompt_token_ids = _normalize_prompt_token_ids( + tokenizer.apply_chat_template(messages, **template_kwargs) + ) + + # Build parsers after rendering, so DeepSeek-V4 can use its custom encoder + # while still sharing the existing Dynamo parser/guided-decoding behavior. tool_call_parser, reasoning_parser = create_parsers( request, tool_call_parser_name=tool_call_parser_name, diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index cb1a73c588d4..ba68a376bf9f 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -11,12 +11,15 @@ import json +import sys +import types import pytest from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.utils.hf_transformers_utils import get_tokenizer +import dynamo.frontend.sglang_prepost as sglang_prepost_module import dynamo.frontend.sglang_processor as sglang_processor_module from dynamo.frontend.sglang_prepost import ( SglangPreprocessResult, @@ -445,6 +448,85 @@ def test_required_tool_choice_builds_json_schema_guidance(self): assert isinstance(guided, dict) assert "json" in guided + def test_required_tool_choice_supports_older_sglang_constraint_signature( + self, monkeypatch + ): + tools = convert_tools( + [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } + ] + ) + + def old_get_json_schema_constraint(sglang_tools, tool_choice): + assert sglang_tools == tools + assert tool_choice == "required" + return {"type": "array", "items": {"type": "object"}} + + monkeypatch.setattr( + sglang_prepost_module, + "get_json_schema_constraint", + old_get_json_schema_constraint, + ) + + guided = build_tool_call_guided_decoding( + {"tool_choice": "required", "parallel_tool_calls": False}, + tool_call_parser_name=None, + sglang_tools=tools, + ) + + assert guided == {"json": {"type": "array", "items": {"type": "object"}}} + + def test_auto_tool_choice_supports_older_structure_constraint_signature( + self, monkeypatch + ): + tools = convert_tools( + [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } + ] + ) + + class OldFunctionCallParser: + def __init__(self, *, tools, tool_call_parser): + self.tools = tools + self.tool_call_parser = tool_call_parser + + def get_structure_constraint(self, tool_choice): + assert tool_choice == "auto" + return "structural_tag", {"type": "object"} + + monkeypatch.setattr( + sglang_prepost_module, + "FunctionCallParser", + OldFunctionCallParser, + ) + + guided = build_tool_call_guided_decoding( + {"tool_choice": "auto", "parallel_tool_calls": False}, + tool_call_parser_name="kimi_k2", + sglang_tools=tools, + ) + + assert guided == {"structural_tag": {"type": "object"}} + def test_auto_strict_tools_can_build_structural_tag_guidance(self): tools = convert_tools( [ @@ -990,6 +1072,240 @@ def test_system_message(self, tokenizer): ) assert len(with_system.prompt_token_ids) > len(without_system.prompt_token_ids) + def test_deepseek_v4_uses_sglang_encoder_when_chat_template_missing( + self, monkeypatch + ): + """DeepSeek-V4 uses SGLang's encoder instead of HF chat_template.""" + captured = {} + fake_module = types.ModuleType("sglang.srt.entrypoints.openai.encoding_dsv4") + + def fake_encode_messages(messages, *, thinking_mode, reasoning_effort=None): + captured["messages"] = messages + captured["thinking_mode"] = thinking_mode + captured["reasoning_effort"] = reasoning_effort + return "" + + fake_module.encode_messages = fake_encode_messages + monkeypatch.setitem( + sys.modules, + "sglang.srt.entrypoints.openai.encoding_dsv4", + fake_module, + ) + + class NoTemplateTokenizer: + chat_template = None + + def apply_chat_template(self, *args, **kwargs): + raise AssertionError("apply_chat_template should not be called") + + def encode(self, prompt): + assert prompt == "" + return [1, 2, 3] + + request = { + "model": "deepseek-ai/DeepSeek-V4-Pro", + "messages": [{"role": "user", "content": "Hello"}], + "chat_template_kwargs": { + "thinking": True, + "reasoning_effort": "max", + }, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + }, + } + ], + } + + result = preprocess_chat_request( + request, + tokenizer=NoTemplateTokenizer(), + tool_call_parser_name=None, + reasoning_parser_name="deepseek_v4", + ) + + assert result.prompt_token_ids == [1, 2, 3] + assert captured["thinking_mode"] == "thinking" + assert captured["reasoning_effort"] == "max" + assert captured["messages"][0]["role"] == "system" + assert captured["messages"][0]["tools"][0]["function"]["name"] == "get_weather" + assert captured["messages"][1]["role"] == "user" + + def test_deepseek_v4_named_tool_choice_filters_encoder_tools(self, monkeypatch): + captured = {} + fake_module = types.ModuleType("sglang.srt.entrypoints.openai.encoding_dsv4") + + def fake_encode_messages(messages, *, thinking_mode, reasoning_effort=None): + captured["messages"] = messages + return "" + + fake_module.encode_messages = fake_encode_messages + monkeypatch.setitem( + sys.modules, + "sglang.srt.entrypoints.openai.encoding_dsv4", + fake_module, + ) + + class NoTemplateTokenizer: + chat_template = None + + def encode(self, prompt): + return [1] + + request = { + "model": "deepseek-ai/DeepSeek-V4-Pro", + "messages": [{"role": "user", "content": "Hello"}], + "tools": [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {}}, + }, + { + "type": "function", + "function": {"name": "get_time", "parameters": {}}, + }, + ], + "tool_choice": { + "type": "function", + "function": {"name": "get_time"}, + }, + } + + preprocess_chat_request( + request, + tokenizer=NoTemplateTokenizer(), + tool_call_parser_name=None, + reasoning_parser_name="deepseek_v4", + ) + + tools = captured["messages"][0]["tools"] + assert [tool["function"]["name"] for tool in tools] == ["get_time"] + + def test_deepseek_v4_respects_existing_chat_template(self, monkeypatch): + fake_module = types.ModuleType("sglang.srt.entrypoints.openai.encoding_dsv4") + + def fake_encode_messages(messages, *, thinking_mode, reasoning_effort=None): + raise AssertionError("encoding_dsv4 should not be called") + + fake_module.encode_messages = fake_encode_messages + monkeypatch.setitem( + sys.modules, + "sglang.srt.entrypoints.openai.encoding_dsv4", + fake_module, + ) + + class TemplateTokenizer: + chat_template = ( + "{% for message in messages %}{{ message.content }}{% endfor %}" + ) + + def apply_chat_template(self, messages, **kwargs): + assert kwargs["add_generation_prompt"] is True + assert kwargs["tokenize"] is True + return [4, 5, 6] + + def encode(self, prompt): + raise AssertionError("encode should not be called") + + result = preprocess_chat_request( + { + "model": "deepseek-ai/DeepSeek-V4-Pro", + "messages": [{"role": "user", "content": "Hello"}], + }, + tokenizer=TemplateTokenizer(), + tool_call_parser_name=None, + reasoning_parser_name=None, + ) + + assert result.prompt_token_ids == [4, 5, 6] + + def test_deepseek_v4_normalizes_none_content_without_mutating_request( + self, monkeypatch + ): + captured = {} + fake_module = types.ModuleType("sglang.srt.entrypoints.openai.encoding_dsv4") + + def fake_encode_messages(messages, *, thinking_mode, reasoning_effort=None): + captured["messages"] = messages + return "" + + fake_module.encode_messages = fake_encode_messages + monkeypatch.setitem( + sys.modules, + "sglang.srt.entrypoints.openai.encoding_dsv4", + fake_module, + ) + + class NoTemplateTokenizer: + chat_template = None + + def encode(self, prompt): + return [7] + + request = { + "model": "deepseek-ai/DeepSeek-V4-Pro", + "messages": [{"role": "assistant", "content": None}], + } + + result = preprocess_chat_request( + request, + tokenizer=NoTemplateTokenizer(), + tool_call_parser_name=None, + reasoning_parser_name=None, + ) + + assert result.prompt_token_ids == [7] + assert captured["messages"] == [{"role": "assistant", "content": ""}] + assert request["messages"] == [{"role": "assistant", "content": None}] + + def test_deepseek_v4_tool_choice_none_strips_encoder_tools(self, monkeypatch): + captured = {} + fake_module = types.ModuleType("sglang.srt.entrypoints.openai.encoding_dsv4") + + def fake_encode_messages(messages, *, thinking_mode, reasoning_effort=None): + captured["messages"] = messages + return "" + + fake_module.encode_messages = fake_encode_messages + monkeypatch.setitem( + sys.modules, + "sglang.srt.entrypoints.openai.encoding_dsv4", + fake_module, + ) + + class NoTemplateTokenizer: + chat_template = None + + def encode(self, prompt): + return [8] + + preprocess_chat_request( + { + "model": "deepseek-ai/DeepSeek-V4-Pro", + "messages": [{"role": "system", "content": "Stay terse."}], + "tools": [ + { + "type": "function", + "function": {"name": "get_weather", "parameters": {}}, + } + ], + "tool_choice": "none", + }, + tokenizer=NoTemplateTokenizer(), + tool_call_parser_name=None, + reasoning_parser_name=None, + exclude_tools_when_tool_choice_none=True, + ) + + assert "tools" not in captured["messages"][0] + # --------------------------------------------------------------------------- # SglangStreamingPostProcessor: incremental detokenization diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 1c5c2b11f183..791793845ae3 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -15,13 +15,96 @@ fallback and any associated polyfills. """ +import inspect import ipaddress import logging import socket +from functools import lru_cache from typing import Any logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Top-level sglang exports: Engine, ServerArgs +# +# Some SGLang dev builds (including 0.5.x snapshots) do not re-export these +# from sglang/__init__.py, while Dynamo historically uses `import sglang as sgl` +# followed by `sgl.Engine(...)` throughout this backend. +# --------------------------------------------------------------------------- +def ensure_sglang_top_level_exports() -> None: + """Restore top-level SGLang exports omitted by some install flavors.""" + import sglang as sgl + + if not hasattr(sgl, "Engine"): + from sglang.srt.entrypoints.engine import Engine + + sgl.Engine = Engine + + if not hasattr(sgl, "ServerArgs"): + from sglang.srt.server_args import ServerArgs + + sgl.ServerArgs = ServerArgs + + +ensure_sglang_top_level_exports() + + +@lru_cache(maxsize=32) +def _get_async_generate_supported_kwarg_names( + async_generate: Any, +) -> frozenset[str] | None: + """Return supported async_generate keyword names, or None for **kwargs.""" + try: + signature = inspect.signature(async_generate) + except (TypeError, ValueError): + logger.debug( + "Could not inspect SGLang Engine.async_generate signature; " + "dropping optional compatibility kwargs" + ) + return frozenset() + + names: set[str] = set() + for name, param in signature.parameters.items(): + if param.kind == inspect.Parameter.VAR_KEYWORD: + return None + if param.kind in ( + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.KEYWORD_ONLY, + ): + names.add(name) + + return frozenset(names) + + +def filter_supported_async_generate_kwargs( + engine: Any, kwargs: dict[str, Any] +) -> dict[str, Any]: + """Return only async_generate kwargs accepted by this SGLang engine. + + SGLang occasionally adds optional Engine.async_generate kwargs before every + supported install flavor has them. Keep the compatibility boundary narrow: + callers decide which kwargs are optional, and this helper only drops those + optional kwargs when the installed engine cannot accept them. + """ + async_generate = engine.async_generate + signature_source = getattr(async_generate, "__func__", async_generate) + + try: + supported_kwarg_names = _get_async_generate_supported_kwarg_names( + signature_source + ) + except TypeError: + supported_kwarg_names = _get_async_generate_supported_kwarg_names.__wrapped__( + signature_source + ) + + if supported_kwarg_names is None: + return kwargs + + return {key: value for key, value in kwargs.items() if key in supported_kwarg_names} + + # --------------------------------------------------------------------------- # Network utilities: NetworkAddress, get_local_ip_auto, get_zmq_socket # @@ -201,6 +284,8 @@ def enable_disjoint_streaming_output(server_args: Any) -> None: __all__ = [ "NetworkAddress", "enable_disjoint_streaming_output", + "ensure_sglang_top_level_exports", + "filter_supported_async_generate_kwargs", "get_local_ip_auto", "get_scheduler_info", "get_zmq_socket", diff --git a/components/src/dynamo/sglang/publisher.py b/components/src/dynamo/sglang/publisher.py index ac250b4eb555..79dafbc5cf5c 100644 --- a/components/src/dynamo/sglang/publisher.py +++ b/components/src/dynamo/sglang/publisher.py @@ -1,6 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import asyncio import json import logging diff --git a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py index 81579d3fba9d..e859eb48fd03 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py @@ -13,6 +13,7 @@ from dynamo.common.constants import DisaggregationMode from dynamo.common.utils.engine_response import normalize_finish_reason from dynamo.common.utils.otel_tracing import build_trace_headers +from dynamo.sglang._compat import filter_supported_async_generate_kwargs from dynamo.sglang.args import Config from dynamo.sglang.publisher import DynamoSglangPublisher from dynamo.sglang.request_handlers.handler_base import BaseWorkerHandler @@ -275,6 +276,9 @@ async def generate( return_routed_experts = getattr( self.config.server_args, "enable_return_routed_experts", False ) + routed_experts_kwargs = filter_supported_async_generate_kwargs( + self.engine, {"return_routed_experts": return_routed_experts} + ) priority = (request.get("routing") or {}).get("priority") logprob_kwargs = self._build_logprob_kwargs(request) @@ -308,7 +312,7 @@ async def generate( **input_param, sampling_params=sampling_params, stream=True, - return_routed_experts=return_routed_experts, + **routed_experts_kwargs, bootstrap_host=bootstrap_info["bootstrap_host"], bootstrap_port=bootstrap_info["bootstrap_port"], bootstrap_room=bootstrap_info["bootstrap_room"], @@ -346,7 +350,7 @@ async def generate( video_data=video_data, sampling_params=sampling_params, stream=True, - return_routed_experts=return_routed_experts, + **routed_experts_kwargs, external_trace_header=trace_header, rid=trace_id, data_parallel_rank=dp_rank, diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py index ee557eef6b50..dd4691fec49e 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_unit.py +++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py @@ -11,6 +11,11 @@ import yaml from sglang.srt.disaggregation.utils import FAKE_BOOTSTRAP_HOST +import dynamo.sglang._compat as sglang_compat +from dynamo.sglang._compat import ( + ensure_sglang_top_level_exports, + filter_supported_async_generate_kwargs, +) from dynamo.sglang.args import parse_args from dynamo.sglang.health_check import ( SglangDisaggHealthCheckPayload, @@ -38,6 +43,99 @@ mock_sglang_cli = make_cli_args_fixture("dynamo.sglang") +def test_compat_restores_sglang_top_level_exports(): + """Dynamo supports SGLang builds that omit top-level Engine/ServerArgs.""" + import sglang as sgl + from sglang.srt.entrypoints.engine import Engine + from sglang.srt.server_args import ServerArgs + + missing = object() + original_engine = getattr(sgl, "Engine", missing) + original_server_args = getattr(sgl, "ServerArgs", missing) + + try: + if hasattr(sgl, "Engine"): + delattr(sgl, "Engine") + if hasattr(sgl, "ServerArgs"): + delattr(sgl, "ServerArgs") + + ensure_sglang_top_level_exports() + + assert sgl.Engine is Engine + assert sgl.ServerArgs is ServerArgs + finally: + if original_engine is missing: + if hasattr(sgl, "Engine"): + delattr(sgl, "Engine") + else: + sgl.Engine = original_engine + + if original_server_args is missing: + if hasattr(sgl, "ServerArgs"): + delattr(sgl, "ServerArgs") + else: + sgl.ServerArgs = original_server_args + + +def test_compat_filters_async_generate_kwargs_for_older_engines(): + class OldEngine: + async def async_generate(self, input_ids=None, sampling_params=None): + return None + + kwargs = { + "input_ids": [1, 2, 3], + "return_routed_experts": True, + } + + assert filter_supported_async_generate_kwargs(OldEngine(), kwargs) == { + "input_ids": [1, 2, 3] + } + + +def test_compat_keeps_async_generate_kwargs_for_newer_engines(): + class NewEngine: + async def async_generate(self, return_routed_experts=False): + return None + + kwargs = {"return_routed_experts": True} + + assert filter_supported_async_generate_kwargs(NewEngine(), kwargs) == kwargs + + +def test_compat_keeps_async_generate_kwargs_for_variadic_engines(): + class VariadicEngine: + async def async_generate(self, **kwargs): + return None + + kwargs = {"return_routed_experts": True} + + assert filter_supported_async_generate_kwargs(VariadicEngine(), kwargs) == kwargs + + +def test_compat_caches_async_generate_signature_inspection(monkeypatch): + class CachedEngine: + async def async_generate(self, return_routed_experts=False): + return None + + sglang_compat._get_async_generate_supported_kwarg_names.cache_clear() + calls = 0 + original_signature = sglang_compat.inspect.signature + + def counting_signature(obj): + nonlocal calls + calls += 1 + return original_signature(obj) + + monkeypatch.setattr(sglang_compat.inspect, "signature", counting_signature) + + kwargs = {"return_routed_experts": True} + assert filter_supported_async_generate_kwargs(CachedEngine(), kwargs) == kwargs + assert filter_supported_async_generate_kwargs(CachedEngine(), kwargs) == kwargs + assert calls == 1 + + sglang_compat._get_async_generate_supported_kwarg_names.cache_clear() + + @pytest.mark.asyncio async def test_custom_jinja_template_invalid_path(mock_sglang_cli): """Test that invalid file path raises FileNotFoundError.""" diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index eb312cf91e90..851c143f9e24 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -1227,8 +1227,9 @@ impl OpenAIPreprocessor { /// For kimi_k25: disabled when chat_template_args contains "thinking": false. /// For nemotron_nano: disabled when chat_template_args contains "enable_thinking": false /// or "force_nonempty_content": true. - /// For deepseek_r1: disabled when chat_template_args contains "thinking": false - /// or "thinking_mode": "chat". + /// For deepseek_r1 / deepseek_v4: disabled when chat_template_args contains + /// "thinking": false or "thinking_mode": "chat" — matches the V4 formatter's + /// `resolve_thinking_mode` convention, so the parser and the prompt stay in sync. fn is_reasoning_disabled_by_request( reasoning_parser: Option<&str>, chat_template_args: Option<&std::collections::HashMap>, @@ -1257,7 +1258,8 @@ impl OpenAIPreprocessor { } false } - Some("deepseek_r1") => { + Some("deepseek_r1") | Some("deepseek_v4") | Some("deepseek-v4") + | Some("deepseekv4") => { if let Some(args) = chat_template_args { if let Some(thinking) = args.get("thinking") { return thinking == &serde_json::Value::Bool(false); @@ -1829,6 +1831,50 @@ mod tests { false, "nemotron_nano + empty args → enabled", ), + // deepseek_v4 — same convention as deepseek_r1; verify all three aliases + // (deepseek_v4 / deepseek-v4 / deepseekv4) plus both signal keys. + ( + Some("deepseek_v4"), + Some(&thinking_false), + true, + "deepseek_v4 + thinking=false → disabled", + ), + ( + Some("deepseek_v4"), + Some(&thinking_true), + false, + "deepseek_v4 + thinking=true → enabled", + ), + ( + Some("deepseek_v4"), + Some(&thinking_mode_chat), + true, + "deepseek_v4 + thinking_mode=chat → disabled", + ), + ( + Some("deepseek_v4"), + Some(&thinking_mode_thinking), + false, + "deepseek_v4 + thinking_mode=thinking → enabled", + ), + ( + Some("deepseek_v4"), + None, + false, + "deepseek_v4 + no args → enabled", + ), + ( + Some("deepseek-v4"), + Some(&thinking_false), + true, + "deepseek-v4 (hyphen alias) + thinking=false → disabled", + ), + ( + Some("deepseekv4"), + Some(&thinking_mode_chat), + true, + "deepseekv4 (joined alias) + thinking_mode=chat → disabled", + ), ]; for (parser, args, expected, desc) in cases { diff --git a/lib/llm/src/preprocessor/prompt.rs b/lib/llm/src/preprocessor/prompt.rs index 105e67c6c0d8..8b1f5fde437e 100644 --- a/lib/llm/src/preprocessor/prompt.rs +++ b/lib/llm/src/preprocessor/prompt.rs @@ -26,6 +26,7 @@ use std::sync::Arc; use crate::preprocessor::media::MediaDecoder; pub mod deepseek_v32; +pub mod deepseek_v4; mod template; pub use template::{ChatTemplate, ContextMixins}; diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs new file mode 100644 index 000000000000..37c534ed3ef6 --- /dev/null +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -0,0 +1,1271 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! DeepSeek V4 native prompt formatting +//! +//! Native Rust port of DeepSeek V4's chat encoding (encoding_dsv4.py). +//! +//! Reference: DeepSeek-V4-Pro/encoding/encoding_dsv4.py + +use anyhow::{Context, Result}; +use serde_json::Value as JsonValue; + +/// Special tokens for DeepSeek V4 +pub mod tokens { + pub const BOS: &str = "<｜begin▁of▁sentence｜>"; + pub const EOS: &str = "<｜end▁of▁sentence｜>"; + pub const THINKING_START: &str = ""; + pub const THINKING_END: &str = ""; + pub const DSML_TOKEN: &str = "｜DSML｜"; + pub const USER_START: &str = "<｜User｜>"; + pub const ASSISTANT_START: &str = "<｜Assistant｜>"; + pub const LATEST_REMINDER: &str = "<｜latest_reminder｜>"; + + // Quick-instruction task tokens + pub const TASK_ACTION: &str = "<｜action｜>"; + pub const TASK_QUERY: &str = "<｜query｜>"; + pub const TASK_AUTHORITY: &str = "<｜authority｜>"; + pub const TASK_DOMAIN: &str = "<｜domain｜>"; + pub const TASK_TITLE: &str = "<｜title｜>"; + pub const TASK_READ_URL: &str = "<｜read_url｜>"; +} + +/// DSML outer block name for tool-call groups: `<｜DSML｜tool_calls>...`. +const TOOL_CALLS_BLOCK_NAME: &str = "tool_calls"; + +/// Wire-format tags that wrap a tool result inside a user content block. +const TOOL_RESULT_OPEN: &str = ""; +const TOOL_RESULT_CLOSE: &str = ""; + +/// Preamble line that introduces a response-format schema in both the +/// system and developer roles. The `{}` placeholder is filled by the +/// caller with the JSON-serialized schema. +const RESPONSE_FORMAT_PREAMBLE: &str = + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{}"; + +/// Roles whose messages are always kept by `drop_thinking_messages`. All other +/// roles before the last user/developer message are dropped (assistant turns +/// keep their non-`reasoning_content` fields; everything else is removed). +const KEEP_ROLES: &[&str] = &[ + "user", + "system", + "tool", + "latest_reminder", + "direct_search_results", +]; + +/// Placeholder the Python reference inserts when it can't render an +/// unsupported content-block or tool-result item type. Rendered into the +/// assistant-visible prompt so a human can tell something was skipped; +/// dynamo additionally emits a `tracing::warn!` so ops see it too. +const UNSUPPORTED_PLACEHOLDER_FMT: &str = "[Unsupported {}]"; + +const REASONING_EFFORT_MAX: &str = "Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"; + +/// Thinking mode for the model +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ThinkingMode { + Chat, + Thinking, +} + +impl ThinkingMode { + /// Tiny branch-free mapping to a static string. `#[inline]` because this + /// is called from inside the per-message render hot path. + #[inline] + pub fn as_str(&self) -> &'static str { + match self { + ThinkingMode::Chat => "chat", + ThinkingMode::Thinking => "thinking", + } + } +} + +/// Reasoning effort level. `None` conveyed as `Option`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReasoningEffort { + Max, + High, +} + +/// Serialize a JSON value to match Python's `json.dumps(ensure_ascii=False)` spacing. +/// Python's default separators are `(', ', ': ')`; we use a custom `Formatter` +/// so escape sequences inside strings can't confuse state tracking. +/// +/// Returns `Result` so serialization / UTF-8 errors propagate to the caller +/// rather than silently collapsing to `"{}"` (the old error-swallow behavior +/// dropped the entire request payload with no signal up the stack). +fn to_json(value: &JsonValue) -> Result { + use serde::Serialize; + use serde_json::ser::Formatter; + use std::io; + + struct PythonFormatter; + + impl Formatter for PythonFormatter { + fn begin_array_value( + &mut self, + writer: &mut W, + first: bool, + ) -> io::Result<()> { + if first { + Ok(()) + } else { + writer.write_all(b", ") + } + } + + fn begin_object_key( + &mut self, + writer: &mut W, + first: bool, + ) -> io::Result<()> { + if first { + Ok(()) + } else { + writer.write_all(b", ") + } + } + + fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()> { + writer.write_all(b": ") + } + } + + // Size the buffer from a compact pre-serialization. Python-style spacing + // adds exactly one byte per structural separator (`,` → `, `, `:` → `: `), + // which bounds the final length by ~1.125× the compact length. This keeps + // small payloads cheap and avoids the 5–9 reallocations the prior fixed + // 64-byte hint forced on KB-sized tool schemas and response formats. + let compact_len = serde_json::to_string(value) + .context("to_json: compact pre-serialization failed")? + .len(); + let capacity = compact_len.saturating_add(compact_len / 8).max(256); + let mut buf = Vec::with_capacity(capacity); + let mut ser = serde_json::Serializer::with_formatter(&mut buf, PythonFormatter); + value + .serialize(&mut ser) + .context("to_json: Python-formatter serialization failed")?; + String::from_utf8(buf).context("to_json: serialized output is not valid UTF-8") +} + +/// Extract function definitions from OpenAI-format tool list. +fn tools_from_openai_format(tools: &[JsonValue]) -> Vec { + tools + .iter() + .filter_map(|tool| tool.get("function").cloned()) + .collect() +} + +/// Render tool schemas into the system prompt format. +/// +/// Previously built via four sequential `String::replace` passes over a +/// `{placeholder}`-based template, which allocated a fresh copy of the whole +/// (schema-inlined, potentially kB-scale) string on each substitution. A +/// single `format!` with named arguments collapses that to one allocation +/// sized from the final length. +fn render_tools(tools: &[JsonValue]) -> Result { + let tools_json: Vec = tools_from_openai_format(tools) + .iter() + .map(to_json) + .collect::>()?; + let schemas = tools_json.join("\n"); + + Ok(format!( + r#"## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml}tool_calls>" block like the following: + +<{dsml}tool_calls> +<{dsml}invoke name="$TOOL_NAME"> +<{dsml}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml}invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by {think_open}), you MUST output your complete reasoning inside {think_open}...{think_close} BEFORE any tool calls or final response. + +Otherwise, output directly after {think_close} with tool calls or final response. + +### Available Tool Schemas + +{schemas} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +"#, + dsml = tokens::DSML_TOKEN, + think_open = tokens::THINKING_START, + think_close = tokens::THINKING_END, + schemas = schemas, + )) +} + +/// Find the index of the last user/developer message. +/// +/// Returns `None` when no such message exists. Callers should treat `None` +/// as Python's `-1` sentinel: `idx >= -1` is always true in Python, so use +/// `Option::is_none_or(|u| idx >= u)` (or `>`) to match the reference encoder. +fn find_last_user_index(messages: &[JsonValue]) -> Option { + messages + .iter() + .enumerate() + .rev() + .find(|(_, msg)| { + msg.get("role") + .and_then(JsonValue::as_str) + .is_some_and(|r| matches!(r, "user" | "developer")) + }) + .map(|(idx, _)| idx) +} + +/// Extract visible text from OpenAI-style message content. +/// +/// Returns `Result` because the `_ => to_json(content)` fallback can now fail +/// (to_json itself returns `Result`). Callers already sit in `Result` context. +fn extract_visible_text(content: &JsonValue) -> Result { + Ok(match content { + JsonValue::String(text) => text.clone(), + JsonValue::Array(items) => items + .iter() + .filter_map(|item| { + if let Some(text) = item.as_str() { + return Some(text.to_string()); + } + let item_type = item.get("type").and_then(JsonValue::as_str); + if item_type == Some("text") { + return item + .get("text") + .and_then(JsonValue::as_str) + .map(|text| text.to_string()); + } + tracing::warn!( + chunk_type = item_type.unwrap_or("unknown"), + "DeepSeek V4 formatter dropped non-text content chunk while normalizing message content", + ); + None + }) + .collect::(), + _ => to_json(content)?, + }) +} + +/// Normalize message `content` fields for text-only DeepSeek V4 rendering. +fn normalize_message_contents(messages: &mut [JsonValue]) -> Result<()> { + for msg in messages { + let Some(content) = msg.get("content") else { + continue; + }; + // Leave non-string/non-array content untouched (null, etc.) + if !content.is_string() && !content.is_array() { + continue; + } + let normalized = extract_visible_text(content)?; + if let Some(obj) = msg.as_object_mut() { + obj.insert("content".to_string(), JsonValue::String(normalized)); + } + } + Ok(()) +} + +/// Encode tool call arguments into DSML parameter format. +fn encode_arguments_to_dsml(tool_call: &JsonValue) -> Result { + let arguments_str = tool_call + .get("arguments") + .and_then(JsonValue::as_str) + .context("Missing or invalid 'arguments' field")?; + + // Python falls back to `{"arguments": raw_string}` on parse failure. + let arguments: JsonValue = match serde_json::from_str(arguments_str) { + Ok(v) => v, + Err(_) => serde_json::json!({ "arguments": arguments_str }), + }; + + let arguments_obj = arguments + .as_object() + .context("Arguments must be a JSON object")?; + + let mut params = Vec::new(); + for (key, value) in arguments_obj { + // Dispatch on the concrete variant so we don't do the `is_string` / `as_str` + // / `unwrap` dance (unwrap is technically safe after is_string but fragile + // against future refactors). + let (is_string, value_str) = match value { + JsonValue::String(s) => (true, s.clone()), + _ => (false, to_json(value)?), + }; + params.push(format!( + "<{}parameter name=\"{}\" string=\"{}\">{}", + tokens::DSML_TOKEN, + key, + if is_string { "true" } else { "false" }, + value_str, + tokens::DSML_TOKEN + )); + } + + Ok(params.join("\n")) +} + +/// Lookup the task token for a quick-instruction task. +/// +/// Called once per assistant-turn in `render_message` and once per transition +/// token lookup; `#[inline]` because the static-str match is smaller than the +/// call overhead. +#[inline] +fn task_token(task: &str) -> Option<&'static str> { + match task { + "action" => Some(tokens::TASK_ACTION), + "query" => Some(tokens::TASK_QUERY), + "authority" => Some(tokens::TASK_AUTHORITY), + "domain" => Some(tokens::TASK_DOMAIN), + "title" => Some(tokens::TASK_TITLE), + "read_url" => Some(tokens::TASK_READ_URL), + _ => None, + } +} + +/// Append the "Response Format" schema block to `prompt` if the message has one. +fn append_response_format(msg: &JsonValue, prompt: &mut String) -> Result<()> { + if let Some(response_format) = msg.get("response_format") { + prompt.push_str("\n\n"); + prompt.push_str(&RESPONSE_FORMAT_PREAMBLE.replace("{}", &to_json(response_format)?)); + } + Ok(()) +} + +/// Append the tools section to `prompt` if the message has a `tools` array. +fn append_tools_section(msg: &JsonValue, prompt: &mut String) -> Result<()> { + if let Some(tools) = msg.get("tools").and_then(JsonValue::as_array) { + prompt.push_str("\n\n"); + prompt.push_str(&render_tools(tools)?); + } + Ok(()) +} + +/// Render the `system` role: raw content, then optional tools + response-format. +fn render_system_role(msg: &JsonValue, prompt: &mut String) -> Result<()> { + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); + prompt.push_str(content); + append_tools_section(msg, prompt)?; + append_response_format(msg, prompt)?; + Ok(()) +} + +/// Render the `developer` role: wraps content in USER_START, then optional +/// tools + response-format. Developer content is required (non-empty). +fn render_developer_role(msg: &JsonValue, prompt: &mut String) -> Result<()> { + let content = msg + .get("content") + .and_then(JsonValue::as_str) + .filter(|s| !s.is_empty()) + .context("Developer role requires content")?; + + prompt.push_str(tokens::USER_START); + prompt.push_str(content); + append_tools_section(msg, prompt)?; + append_response_format(msg, prompt)?; + Ok(()) +} + +/// Render the `user` role: either content-blocks (with tool_result wrapping) +/// or a plain string `content` field. +fn render_user_role(msg: &JsonValue, prompt: &mut String) -> Result<()> { + prompt.push_str(tokens::USER_START); + if let Some(blocks) = msg.get("content_blocks").and_then(JsonValue::as_array) { + let mut parts: Vec = Vec::with_capacity(blocks.len()); + for block in blocks { + let block_type = block.get("type").and_then(JsonValue::as_str).unwrap_or(""); + match block_type { + "text" => { + let text = block.get("text").and_then(JsonValue::as_str).unwrap_or(""); + parts.push(text.to_string()); + } + "tool_result" => { + let rendered = render_tool_result_content( + block.get("content").unwrap_or(&JsonValue::Null), + )?; + parts.push(format!( + "{}{}{}", + TOOL_RESULT_OPEN, rendered, TOOL_RESULT_CLOSE + )); + } + other => { + tracing::warn!( + block_type = other, + "DeepSeek V4 formatter emitted placeholder for unsupported user content block type", + ); + parts.push(UNSUPPORTED_PLACEHOLDER_FMT.replace("{}", other)); + } + } + } + prompt.push_str(&parts.join("\n\n")); + } else { + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); + prompt.push_str(content); + } + Ok(()) +} + +/// Render the `latest_reminder` role: LATEST_REMINDER token + content. +fn render_latest_reminder_role(msg: &JsonValue, prompt: &mut String) { + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); + prompt.push_str(tokens::LATEST_REMINDER); + prompt.push_str(content); +} + +/// Render the `assistant` role: optional thinking prefix, content, optional +/// `tool_calls` DSML block, optional EOS. +/// +/// Needs `index` and `messages` to peek at the previous turn's `task` field +/// (suppresses thinking prefix when the prior message was a task message). +fn render_assistant_role( + msg: &JsonValue, + index: usize, + messages: &[JsonValue], + thinking_mode: ThinkingMode, + drop_thinking: bool, + last_user_idx: Option, + prompt: &mut String, +) -> Result<()> { + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); + let reasoning = msg + .get("reasoning_content") + .and_then(JsonValue::as_str) + .unwrap_or(""); + let wo_eos = msg + .get("wo_eos") + .and_then(JsonValue::as_bool) + .unwrap_or(false); + + let prev_has_task = index > 0 + && messages[index - 1] + .get("task") + .map(|v| !v.is_null()) + .unwrap_or(false); + + if thinking_mode == ThinkingMode::Thinking && !prev_has_task { + let render_thinking = !drop_thinking || last_user_idx.is_none_or(|u| index > u); + if render_thinking { + prompt.push_str(reasoning); + prompt.push_str(tokens::THINKING_END); + } + } + + prompt.push_str(content); + + if let Some(tool_calls) = msg.get("tool_calls").and_then(JsonValue::as_array) + && !tool_calls.is_empty() + { + prompt.push_str("\n\n"); + prompt.push_str(&format!( + "<{}{}>\n", + tokens::DSML_TOKEN, + TOOL_CALLS_BLOCK_NAME + )); + + let mut invocations = Vec::with_capacity(tool_calls.len()); + for tc in tool_calls { + // Accept both OpenAI-format (nested `function`) and internal + // `{name, arguments}` shape, matching Python's `tool_calls_from_openai_format`. + let fn_obj = tc.get("function").unwrap_or(tc); + let name = fn_obj + .get("name") + .and_then(JsonValue::as_str) + .context("Missing tool call name")?; + let arguments = encode_arguments_to_dsml(fn_obj)?; + invocations.push(format!( + "<{}invoke name=\"{}\">\n{}\n", + tokens::DSML_TOKEN, + name, + arguments, + tokens::DSML_TOKEN + )); + } + prompt.push_str(&invocations.join("\n")); + prompt.push_str(&format!( + "\n", + tokens::DSML_TOKEN, + TOOL_CALLS_BLOCK_NAME + )); + } + + if !wo_eos { + prompt.push_str(tokens::EOS); + } + Ok(()) +} + +/// Render a single message at the given index. +fn render_message( + index: usize, + messages: &[JsonValue], + thinking_mode: ThinkingMode, + drop_thinking: bool, + reasoning_effort: Option, + last_user_idx: Option, +) -> Result { + let msg = &messages[index]; + + let role = msg + .get("role") + .and_then(JsonValue::as_str) + .context("Missing 'role' field")?; + + let mut prompt = String::new(); + + // Reasoning effort prefix (only at index 0 in thinking mode with max effort). + if index == 0 + && thinking_mode == ThinkingMode::Thinking + && reasoning_effort == Some(ReasoningEffort::Max) + { + prompt.push_str(REASONING_EFFORT_MAX); + } + + match role { + "system" => render_system_role(msg, &mut prompt)?, + "developer" => render_developer_role(msg, &mut prompt)?, + "user" => render_user_role(msg, &mut prompt)?, + "latest_reminder" => render_latest_reminder_role(msg, &mut prompt), + "tool" => anyhow::bail!( + "deepseek_v4 merges tool messages into user; preprocess with merge_tool_messages()" + ), + "assistant" => render_assistant_role( + msg, + index, + messages, + thinking_mode, + drop_thinking, + last_user_idx, + &mut prompt, + )?, + other => anyhow::bail!("Unknown role: {other}"), + } + + // Early return if the next message is not assistant/latest_reminder — no transition appended. + if index + 1 < messages.len() { + let next_role = messages[index + 1].get("role").and_then(JsonValue::as_str); + if !matches!(next_role, Some("assistant") | Some("latest_reminder")) { + return Ok(prompt); + } + } + + // Transition tokens based on task field and role. + let task = msg.get("task").and_then(JsonValue::as_str); + if let Some(task) = task { + let sp = task_token(task).with_context(|| format!("Invalid task: '{}'", task))?; + if task != "action" { + prompt.push_str(sp); + } else { + prompt.push_str(tokens::ASSISTANT_START); + prompt.push_str(if thinking_mode != ThinkingMode::Thinking { + tokens::THINKING_END + } else { + tokens::THINKING_START + }); + prompt.push_str(sp); + } + } else if matches!(role, "user" | "developer") { + prompt.push_str(tokens::ASSISTANT_START); + let seed_thinking = thinking_mode == ThinkingMode::Thinking + && (!drop_thinking || last_user_idx.is_none_or(|u| index >= u)); + prompt.push_str(if seed_thinking { + tokens::THINKING_START + } else { + tokens::THINKING_END + }); + } + + Ok(prompt) +} + +/// Render a tool_result `content` payload (string or content-block list). +fn render_tool_result_content(content: &JsonValue) -> Result { + Ok(match content { + JsonValue::String(s) => s.clone(), + JsonValue::Array(items) => { + let mut parts: Vec = Vec::with_capacity(items.len()); + for item in items { + let item_type = item.get("type").and_then(JsonValue::as_str).unwrap_or(""); + if item_type == "text" { + parts.push( + item.get("text") + .and_then(JsonValue::as_str) + .unwrap_or("") + .to_string(), + ); + } else { + tracing::warn!( + item_type, + "DeepSeek V4 formatter emitted placeholder for unsupported tool_result content item type", + ); + parts.push(UNSUPPORTED_PLACEHOLDER_FMT.replace("{}", item_type)); + } + } + parts.join("\n\n") + } + JsonValue::Null => String::new(), + _ => to_json(content)?, + }) +} + +/// Merge `tool` role messages into preceding user `content_blocks` and collapse +/// consecutive user turns, matching Python's `merge_tool_messages`. +/// +/// Iterates the input by reference. Each message is cloned at most once — and +/// only when the control flow actually moves it into `merged` (the "other +/// role" pass-through branch). The `tool` and `user` branches extract the few +/// fields they need and build fresh JSON objects, so cloning the whole input +/// message up front (as the original implementation did) was pure overhead +/// on long chat histories. +pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { + let mut merged: Vec = Vec::with_capacity(messages.len()); + + for msg in messages { + let role = msg.get("role").and_then(JsonValue::as_str).unwrap_or(""); + + if role == "tool" { + let tool_block = serde_json::json!({ + "type": "tool_result", + "tool_use_id": msg.get("tool_call_id").cloned().unwrap_or(JsonValue::String(String::new())), + "content": msg.get("content").cloned().unwrap_or(JsonValue::String(String::new())), + }); + + let can_merge = merged + .last() + .map(|m| { + m.get("role").and_then(JsonValue::as_str) == Some("user") + && m.get("content_blocks").is_some() + }) + .unwrap_or(false); + + if can_merge { + // `can_merge` already checked `content_blocks.is_some()`; if + // the subsequent `as_array_mut()` ever fails (data invariant + // violation) we match the original behavior and silently + // drop rather than falling through to push a new user msg. + if let Some(last) = merged.last_mut() + && let Some(blocks) = last + .as_object_mut() + .and_then(|o| o.get_mut("content_blocks")) + .and_then(JsonValue::as_array_mut) + { + blocks.push(tool_block); + } + } else { + merged.push(serde_json::json!({ + "role": "user", + "content_blocks": [tool_block], + })); + } + } else if role == "user" { + let text = msg + .get("content") + .and_then(JsonValue::as_str) + .unwrap_or("") + .to_string(); + let text_block = serde_json::json!({ "type": "text", "text": text }); + + let can_merge = merged + .last() + .map(|m| { + m.get("role").and_then(JsonValue::as_str) == Some("user") + && m.get("content_blocks").is_some() + && m.get("task").map(|v| v.is_null()).unwrap_or(true) + }) + .unwrap_or(false); + + if can_merge { + if let Some(last) = merged.last_mut() + && let Some(blocks) = last + .as_object_mut() + .and_then(|o| o.get_mut("content_blocks")) + .and_then(JsonValue::as_array_mut) + { + blocks.push(text_block); + } + } else { + let mut new_msg = serde_json::json!({ + "role": "user", + "content": text, + "content_blocks": [text_block], + }); + // Preserve extra fields. + if let Some(obj) = new_msg.as_object_mut() { + for key in ["task", "wo_eos", "mask"] { + if let Some(v) = msg.get(key) { + obj.insert(key.to_string(), v.clone()); + } + } + } + merged.push(new_msg); + } + } else { + // Pass-through: clone only when we're actually moving the message. + merged.push(msg.clone()); + } + } + + merged +} + +/// Sort `tool_result` blocks within user messages by the `tool_calls[].id` order +/// of the preceding assistant message. +pub fn sort_tool_results_by_call_order(mut messages: Vec) -> Vec { + use std::collections::HashMap; + let mut last_order: HashMap = HashMap::new(); + + for msg in &mut messages { + let role = msg.get("role").and_then(JsonValue::as_str).unwrap_or(""); + if role == "assistant" { + if let Some(tcs) = msg.get("tool_calls").and_then(JsonValue::as_array) { + last_order.clear(); + for (idx, tc) in tcs.iter().enumerate() { + let id = tc + .get("id") + .and_then(JsonValue::as_str) + .or_else(|| { + tc.get("function") + .and_then(|f| f.get("id")) + .and_then(JsonValue::as_str) + }) + .unwrap_or(""); + if !id.is_empty() { + last_order.insert(id.to_string(), idx); + } + } + } + } else if role == "user" && !last_order.is_empty() { + let Some(blocks) = msg + .as_object_mut() + .and_then(|o| o.get_mut("content_blocks")) + .and_then(JsonValue::as_array_mut) + else { + continue; + }; + + // Collect tool_result blocks with their positions. + let tool_positions: Vec = blocks + .iter() + .enumerate() + .filter(|(_, b)| b.get("type").and_then(JsonValue::as_str) == Some("tool_result")) + .map(|(i, _)| i) + .collect(); + + if tool_positions.len() > 1 { + let mut tool_blocks: Vec = + tool_positions.iter().map(|&i| blocks[i].clone()).collect(); + tool_blocks.sort_by_key(|b| { + let id = b + .get("tool_use_id") + .and_then(JsonValue::as_str) + .unwrap_or(""); + *last_order.get(id).unwrap_or(&0) + }); + for (sorted_idx, &pos) in tool_positions.iter().enumerate() { + blocks[pos] = tool_blocks[sorted_idx].clone(); + } + } + } + } + + messages +} + +/// Drop reasoning and non-essential messages before the last user message. +/// +/// Takes `last_user_idx` (the pre-drop position of the last user/developer +/// message, as returned by [`find_last_user_index`]) from the caller so we +/// don't rescan the vector; returns the post-drop position of the same +/// message so the caller doesn't have to rescan either. +fn drop_thinking_messages( + messages: Vec, + last_user_idx: Option, +) -> (Vec, Option) { + let mut out = Vec::with_capacity(messages.len()); + let mut new_last_user_idx: Option = None; + for (idx, mut msg) in messages.into_iter().enumerate() { + let role = msg.get("role").and_then(JsonValue::as_str).unwrap_or(""); + if KEEP_ROLES.contains(&role) || last_user_idx.is_none_or(|u| idx >= u) { + if last_user_idx == Some(idx) { + new_last_user_idx = Some(out.len()); + } + out.push(msg); + } else if role == "assistant" { + if let Some(obj) = msg.as_object_mut() { + obj.remove("reasoning_content"); + } + out.push(msg); + } + // developer and other roles before last_user_idx are dropped. + } + (out, new_last_user_idx) +} + +/// Encode messages to prompt string with default options. +/// +/// Equivalent to `encode_messages_with_options(.., drop_thinking=true, reasoning_effort=None)`. +pub fn encode_messages( + messages: &[JsonValue], + thinking_mode: ThinkingMode, + add_bos_token: bool, +) -> Result { + encode_messages_with_options(messages, thinking_mode, add_bos_token, true, None) +} + +/// Encode messages to prompt string. +/// +/// # Arguments +/// * `messages` - Array of messages in OpenAI format +/// * `thinking_mode` - Chat or Thinking +/// * `add_bos_token` - Whether to prepend BOS token +/// * `drop_thinking` - Drop reasoning_content from earlier turns (auto-disabled if tools present) +/// * `reasoning_effort` - Optional reasoning effort level (Max prepends a verbatim block) +pub fn encode_messages_with_options( + messages: &[JsonValue], + thinking_mode: ThinkingMode, + add_bos_token: bool, + drop_thinking: bool, + reasoning_effort: Option, +) -> Result { + let merged = merge_tool_messages(messages); + let mut full = sort_tool_results_by_call_order(merged); + + let mut prompt = String::new(); + if add_bos_token { + prompt.push_str(tokens::BOS); + } + + // Auto-disable drop_thinking when any message carries a `tools` field. + let has_tools = full.iter().any(|m| { + m.get("tools") + .map(|v| match v { + JsonValue::Array(a) => !a.is_empty(), + JsonValue::Null => false, + _ => true, + }) + .unwrap_or(false) + }); + let effective_drop_thinking = drop_thinking && !has_tools; + + // Locate the last user/developer message once. `drop_thinking_messages` + // both consumes this (to know what to keep) and returns the new index + // (to avoid a second full scan over its output list). + let mut last_user_idx = find_last_user_index(&full); + + if thinking_mode == ThinkingMode::Thinking && effective_drop_thinking { + let (dropped, new_last_user) = drop_thinking_messages(full, last_user_idx); + full = dropped; + last_user_idx = new_last_user; + } + + for idx in 0..full.len() { + let part = render_message( + idx, + &full, + thinking_mode, + effective_drop_thinking, + reasoning_effort, + last_user_idx, + )?; + prompt.push_str(&part); + } + + Ok(prompt) +} + +/// DeepSeek V4 Prompt Formatter +#[derive(Debug)] +pub struct DeepSeekV4Formatter { + thinking_mode: ThinkingMode, +} + +impl DeepSeekV4Formatter { + pub fn new(thinking_mode: ThinkingMode) -> Self { + Self { thinking_mode } + } + + /// Create formatter with thinking mode enabled (default for DSV4) + pub fn new_thinking() -> Self { + Self::new(ThinkingMode::Thinking) + } + + /// Create formatter with chat mode + pub fn new_chat() -> Self { + Self::new(ThinkingMode::Chat) + } + + fn resolve_thinking_mode( + &self, + args: Option<&std::collections::HashMap>, + ) -> ThinkingMode { + if let Some(args) = args + && let Some(thinking) = args.get("thinking").and_then(JsonValue::as_bool) + { + return if thinking { + ThinkingMode::Thinking + } else { + ThinkingMode::Chat + }; + } + if let Some(args) = args + && let Some(mode) = args.get("thinking_mode").and_then(JsonValue::as_str) + { + match mode { + "chat" => return ThinkingMode::Chat, + "thinking" => return ThinkingMode::Thinking, + _ => {} + } + } + self.thinking_mode + } +} + +impl super::OAIPromptFormatter for DeepSeekV4Formatter { + fn supports_add_generation_prompt(&self) -> bool { + true + } + + fn render(&self, req: &dyn super::OAIChatLikeRequest) -> Result { + let thinking_mode = self.resolve_thinking_mode(req.chat_template_args()); + + let messages_value = req.messages(); + let messages_json = + serde_json::to_value(&messages_value).context("Failed to convert messages to JSON")?; + + let mut messages_array = messages_json + .as_array() + .context("Messages is not an array")? + .clone(); + + normalize_message_contents(&mut messages_array)?; + + let tools_json = req + .tools() + .map(|t| serde_json::to_value(&t)) + .transpose() + .context("Failed to convert tools to JSON")?; + + let response_format_json = req + .response_format() + .map(|rf| serde_json::to_value(&rf)) + .transpose() + .context("Failed to convert response_format to JSON")?; + + if tools_json.is_some() || response_format_json.is_some() { + let system_idx = messages_array + .iter() + .position(|msg| msg.get("role").and_then(JsonValue::as_str) == Some("system")); + + if let Some(idx) = system_idx { + if let Some(msg) = messages_array.get_mut(idx) + && let Some(obj) = msg.as_object_mut() + { + if let Some(tools) = tools_json { + obj.insert("tools".to_string(), tools); + } + if let Some(rf) = response_format_json { + obj.insert("response_format".to_string(), rf); + } + } + } else { + let mut system_msg = serde_json::json!({ + "role": "system", + "content": "" + }); + if let Some(obj) = system_msg.as_object_mut() { + if let Some(tools) = tools_json { + obj.insert("tools".to_string(), tools); + } + if let Some(rf) = response_format_json { + obj.insert("response_format".to_string(), rf); + } + } + messages_array.insert(0, system_msg); + } + } + + encode_messages(&messages_array, thinking_mode, true) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_simple_conversation() { + let messages = json!([ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "reasoning_content": "greet", "content": "Hi!"}, + {"role": "user", "content": "What is 2+2?"} + ]); + let out = + encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true).unwrap(); + assert!(out.starts_with(tokens::BOS)); + assert!(out.ends_with(&format!( + "{}{}", + tokens::ASSISTANT_START, + tokens::THINKING_START + ))); + // drop_thinking default true → earlier reasoning stripped + assert!(!out.contains("greet")); + } + + #[test] + fn test_reasoning_effort_max_prefix() { + let messages = json!([ + {"role": "system", "content": "hi"}, + {"role": "user", "content": "hello"} + ]); + let out = encode_messages_with_options( + messages.as_array().unwrap(), + ThinkingMode::Thinking, + true, + true, + Some(ReasoningEffort::Max), + ) + .unwrap(); + assert!(out.contains("Reasoning Effort: Absolute maximum")); + // Prefix comes between BOS and system content. + let after_bos = &out[tokens::BOS.len()..]; + assert!(after_bos.starts_with("Reasoning Effort:")); + + // High and None do not emit the prefix. + let out2 = encode_messages_with_options( + messages.as_array().unwrap(), + ThinkingMode::Thinking, + true, + true, + Some(ReasoningEffort::High), + ) + .unwrap(); + assert!(!out2.contains("Reasoning Effort: Absolute maximum")); + } + + #[test] + fn test_content_blocks_with_tool_result() { + // `merge_tool_messages` turns a `tool` role followed by a plain user text + // into a single user turn whose `content_blocks` interleave the tool result + // with the text, joined by "\n\n" at render time. Users don't construct + // `content_blocks` directly — both the Python reference and this port + // overwrite any user-supplied `content_blocks` with a single text block. + let messages = json!([ + {"role": "user", "content": "call tool"}, + {"role": "assistant", "content": "", "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": "f", "arguments": "{}"} + }]}, + {"role": "tool", "tool_call_id": "c1", "content": "RESULT"}, + {"role": "user", "content": "thanks"} + ]); + let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Chat, true).unwrap(); + assert!( + out.contains("RESULT\n\nthanks"), + "expected tool_result block followed by 'thanks' in the merged user turn, got:\n{}", + out + ); + } + + #[test] + fn test_drop_thinking_auto_disable_when_tools_present() { + let messages = json!([ + {"role": "system", "content": "s", "tools": [{ + "type": "function", + "function": {"name": "f", "description": "", "parameters": {"type": "object", "properties": {}}} + }]}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "reasoning_content": "PRIOR_REASONING", "content": "reply"}, + {"role": "user", "content": "again"} + ]); + let out = + encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true).unwrap(); + // Tools present → drop_thinking auto-disabled → earlier reasoning preserved. + assert!(out.contains("PRIOR_REASONING")); + } + + // ---- Regression tests for known divergences from the Python reference ---- + + /// Bug: `last_user_idx = None` (no user/developer in history) should behave + /// like Python's `-1` sentinel — `index >= -1` / `idx >= -1` always true, so + /// earlier reasoning is preserved and the assistant's reasoning block is + /// rendered. Rust defaulting `None` to `usize::MAX` / `is_some_and` silently + /// stripped reasoning instead. + /// + /// Byte-equivalent to Python reference with the same input: + /// `sysREASONING_BLOCKhello` + #[test] + fn test_assistant_reasoning_preserved_when_no_user_in_history() { + let messages = json!([ + {"role": "system", "content": "sys"}, + {"role": "assistant", "content": "hello", "reasoning_content": "REASONING_BLOCK"} + ]); + let out = + encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true).unwrap(); + assert_eq!( + out, "<｜begin▁of▁sentence｜>sysREASONING_BLOCKhello<｜end▁of▁sentence｜>", + "Output must match Python reference byte-for-byte when no user/developer in history" + ); + } + + /// Bug: `to_json` tracks in-string state via `prev_char != '\\'` which + /// mis-handles consecutive backslashes. A value containing `\\` (one literal + /// backslash in JSON) makes the helper think the closing `"` is escaped, + /// so it stops inserting Python-compatible spaces after subsequent `:`/`,`. + /// + /// Python `json.dumps({"path": "\\", "count": 5}, ensure_ascii=False)` + /// emits `{"path": "\\", "count": 5}` — space after every `:` and `,`. + #[test] + fn test_to_json_preserves_spacing_past_escaped_backslash() { + let v = json!({"path": "\\", "count": 5}); + let got = to_json(&v).expect("to_json must succeed on well-formed input"); + assert_eq!( + got, r#"{"path": "\\", "count": 5}"#, + "to_json must match Python's json.dumps formatting past an escaped backslash" + ); + } + + /// Larger-than-64-byte inputs (typical tool schemas / response formats) + /// must round-trip unchanged — pins that the capacity hint doesn't + /// truncate and that Python spacing holds across deep nesting. + #[test] + fn test_to_json_handles_large_payload() { + // Build a ~5 KB tool-like schema by nesting an array of items. + let items: Vec = (0..200) + .map(|i| json!({"name": format!("field_{i}"), "type": "string", "i": i})) + .collect(); + let v = json!({ + "type": "object", + "properties": {"items": {"type": "array", "items": items}}, + "required": ["items"], + }); + + let got = to_json(&v).expect("to_json must succeed on well-formed input"); + // Baseline: default serde_json::to_string round-trips. + let parsed: serde_json::Value = serde_json::from_str(&got).expect("round-trip parse"); + assert_eq!( + parsed, v, + "to_json output must round-trip back to the input" + ); + // Python spacing assertions: no bare `",` or `":` sequences outside of + // string literals. The payload contains no commas or colons inside + // string values, so a byte scan is sufficient. + assert!( + !got.contains("\",\""), + "expected ', ' between keys — raw '\",\"' should not appear", + ); + assert!( + !got.contains("\":\""), + "expected ': ' between key and value — raw '\":\"' should not appear", + ); + // Sanity: large payload exercised (> 5KB). + assert!( + got.len() > 5_000, + "test payload is too small: {}", + got.len() + ); + } + + /// `drop_thinking_messages` now takes the pre-drop last-user index and + /// returns the post-drop index instead of forcing the caller to rescan + /// the output list. This test pins that the returned index is + /// equivalent to a full `find_last_user_index` rescan, for the shape + /// that actually shifts indices (non-KEEP role dropped before the + /// final user/developer). + #[test] + fn test_drop_thinking_messages_returns_post_drop_last_user_idx() { + // A `developer` message before the last user triggers an index + // shift: it's not in KEEP and is at idx < last_user_idx, so it + // gets dropped and every surviving message's index decreases by 1. + let messages = vec![ + json!({"role": "developer", "content": "dev1"}), + json!({"role": "assistant", "reasoning_content": "r", "content": "a1"}), + json!({"role": "user", "content": "u1"}), + json!({"role": "developer", "content": "dev2"}), + ]; + let pre_drop_idx = find_last_user_index(&messages); + // The last user-like message is the trailing developer at idx 3. + assert_eq!(pre_drop_idx, Some(3)); + + let (dropped, new_idx) = drop_thinking_messages(messages, pre_drop_idx); + + // developer at idx 0: dropped (not KEEP, before last_user_idx). + // assistant at idx 1: kept with reasoning stripped. + // user at idx 2: kept. + // developer at idx 3: kept (is last_user_idx). + assert_eq!(dropped.len(), 3, "expected 3 messages after drop"); + assert!( + dropped[0].get("reasoning_content").is_none(), + "assistant reasoning_content must be stripped", + ); + assert_eq!( + dropped[0].get("role").and_then(JsonValue::as_str), + Some("assistant"), + ); + assert_eq!( + dropped[1].get("role").and_then(JsonValue::as_str), + Some("user") + ); + assert_eq!( + dropped[2].get("role").and_then(JsonValue::as_str), + Some("developer"), + ); + + // The core invariant: the returned post-drop index is identical to + // what a full `find_last_user_index` rescan would produce. This + // is what allows `encode_messages_with_options` to skip the + // previously-redundant second scan. + let rescan_idx = find_last_user_index(&dropped); + assert_eq!( + new_idx, rescan_idx, + "drop_thinking_messages must return the same index find_last_user_index would compute on its output", + ); + assert_eq!(new_idx, Some(2), "developer shifted from idx 3 to idx 2"); + } + + /// Sanity check for the no-shift case: when nothing before the last + /// user/developer is droppable, the index returned by + /// `drop_thinking_messages` must equal the input index. + #[test] + fn test_drop_thinking_messages_preserves_idx_when_nothing_dropped() { + let messages = vec![ + json!({"role": "system", "content": "s"}), + json!({"role": "user", "content": "u1"}), + json!({"role": "assistant", "reasoning_content": "r", "content": "a"}), + json!({"role": "user", "content": "u2"}), + ]; + let pre_drop_idx = find_last_user_index(&messages); + assert_eq!(pre_drop_idx, Some(3)); + + let (dropped, new_idx) = drop_thinking_messages(messages, pre_drop_idx); + assert_eq!(dropped.len(), 4, "nothing should be dropped here"); + // No shift: last user still at the same index. + assert_eq!(new_idx, Some(3)); + assert_eq!(new_idx, find_last_user_index(&dropped)); + } + + /// Edge case: no user/developer anywhere in the history. Both the + /// pre-drop and post-drop indices must be `None` so the render loop + /// treats every message as "after last user" (Python `-1` sentinel). + #[test] + fn test_drop_thinking_messages_no_user_in_history() { + let messages = vec![ + json!({"role": "system", "content": "s"}), + json!({"role": "assistant", "reasoning_content": "r", "content": "a"}), + ]; + let pre_drop_idx = find_last_user_index(&messages); + assert_eq!(pre_drop_idx, None); + + let (dropped, new_idx) = drop_thinking_messages(messages, pre_drop_idx); + // With `last_user_idx == None`, the `idx >= u` guard is vacuously + // true for every index, so the assistant is kept (with reasoning). + assert_eq!(dropped.len(), 2); + assert_eq!(new_idx, None); + } +} diff --git a/lib/llm/src/preprocessor/prompt/template.rs b/lib/llm/src/preprocessor/prompt/template.rs index fbdf2da1af4a..b9b86c599f2c 100644 --- a/lib/llm/src/preprocessor/prompt/template.rs +++ b/lib/llm/src/preprocessor/prompt/template.rs @@ -19,12 +19,35 @@ use tokcfg::ChatTemplateValue; impl PromptFormatter { pub fn from_mdc(mdc: &ModelDeploymentCard) -> Result { - // Special handling for DeepSeek-V3.2(-Speciale) which doesn't provide Jinja chat_template - let name_lower = mdc.display_name.to_lowercase(); - if name_lower.contains("deepseek") - && name_lower.contains("v3.2") - && !name_lower.contains("exp") - { + // Special handling for DeepSeek models whose HF repos don't ship a Jinja chat_template. + // + // Prefer the authoritative `model_type` from config.json — it's set by + // the model author and survives any `--served-model-name` rename. Fall + // back to a tight substring match on `display_name` only when config.json + // is absent (e.g., tokenizer-only MDCs) or unreadable. + // + // An empty `model_type` string (rare but legal in the JSON) carries + // no signal — normalize it to `None` so the display-name fallback + // still runs instead of being silently suppressed. + let model_type_lower = mdc + .model_info + .as_ref() + .and_then(|info| info.get_model_info().ok()) + .map(|info| info.model_type().to_lowercase()) + .filter(|s| !s.is_empty()); + let display_name_lower = mdc.display_name.to_lowercase(); + + if is_deepseek_v4(&model_type_lower, &display_name_lower) { + tracing::info!( + model_type = ?model_type_lower, + display_name = %mdc.display_name, + "Detected DeepSeek V4 model, using native Rust formatter", + ); + return Ok(Self::OAI(Arc::new( + super::deepseek_v4::DeepSeekV4Formatter::new_thinking(), + ))); + } + if is_deepseek_v3_2_non_exp(&model_type_lower, &display_name_lower) { tracing::info!("Detected DeepSeek V3.2 model (non-Exp), using native Rust formatter"); return Ok(Self::OAI(Arc::new( super::deepseek_v32::DeepSeekV32Formatter::new_thinking(), @@ -187,3 +210,158 @@ struct HfTokenizerConfigJsonFormatter { pub struct ContextMixins { context_mixins: HashSet, } + +/// Decides whether to activate the DeepSeek-V4 native formatter. +/// +/// Primary signal: config.json `model_type`. DeepSeek-V4-Pro and V4-Flash both +/// ship `"model_type": "deepseek_v4"`, set by the model author — this survives +/// any `--served-model-name` rename. +/// +/// Fallback: `display_name`, tight-matched against +/// `^deepseek(?:[-_.])?v4(?:[-_.]|$)`. Only consulted when config.json is +/// absent (tokenizer-only MDCs) or unreadable; a concrete config.json value +/// that is *not* `deepseek_v4` is authoritative and suppresses the fallback. +fn is_deepseek_v4(model_type_lower: &Option, display_name_lower: &str) -> bool { + match model_type_lower.as_deref() { + Some("deepseek_v4") => true, + Some(_) => false, // config.json says something else — trust it + None => is_deepseek_v4_name(display_name_lower), + } +} + +/// Decides whether to activate the DeepSeek-V3.2 (non-Exp) native formatter. +/// Same config-primary / name-fallback rule as V4. +fn is_deepseek_v3_2_non_exp(model_type_lower: &Option, display_name_lower: &str) -> bool { + let name_match = display_name_lower.contains("deepseek") + && display_name_lower.contains("v3.2") + && !display_name_lower.contains("exp"); + match model_type_lower.as_deref() { + Some("deepseek_v3_2") => !display_name_lower.contains("exp"), + Some(_) => false, + None => name_match, + } +} + +/// Tight, anchored match for DeepSeek-V4 display names. Equivalent to the +/// regex `^deepseek(?:[-_.])?v4(?:[-_.]|$)` over an already-lowercased string. +/// Written with string ops to avoid pulling in the `regex` crate. +/// +/// Rejects composite names that previously short-circuited the V4 branch: +/// - `deepseek-v3.2-v4-foo` (the `v3.2` variant is the real one) +/// - `deepseek-v40` / `deepseek-v4pro` (no separator after `v4`) +/// - `my-deepseek-v4` (prefix must be at the start) +fn is_deepseek_v4_name(name_lower: &str) -> bool { + let Some(rest) = name_lower.strip_prefix("deepseek") else { + return false; + }; + // Optional single separator between "deepseek" and "v4". + let rest = rest + .strip_prefix(|c: char| matches!(c, '-' | '_' | '.')) + .unwrap_or(rest); + let Some(after_v4) = rest.strip_prefix("v4") else { + return false; + }; + // `v4` must end the name or be followed by a separator — anything else + // (e.g. `v40`, `v4pro`) is a different model family. + after_v4.is_empty() || after_v4.starts_with(['-', '_', '.']) +} + +#[cfg(test)] +mod detection_tests { + use super::{is_deepseek_v3_2_non_exp, is_deepseek_v4, is_deepseek_v4_name}; + + #[test] + fn v4_name_matches_canonical_variants() { + for name in [ + "deepseek-v4", + "deepseek_v4", + "deepseek.v4", + "deepseekv4", + "deepseek-v4-pro", + "deepseek-v4-flash", + "deepseek-v4-flash-2507", + "deepseek-v4.1", + "deepseek_v4_thinking", + ] { + assert!(is_deepseek_v4_name(name), "expected {name} to match V4"); + } + } + + #[test] + fn v4_name_rejects_non_v4() { + // Composite names that previously short-circuited to V4 before the + // V3.2 branch — now correctly rejected. + for name in [ + "deepseek-v3.2-v4-foo", + "my-deepseek-v4", + "deepseek-v40", + "deepseek-v4pro", + "deepseekv40", + "deepseek-v3", + "deepseek-v3.2", + "deepseek-r1", + "qwen3-v4", // only deepseek-prefixed names qualify + "dsflash", + "", + ] { + assert!( + !is_deepseek_v4_name(name), + "expected {name} to NOT match V4", + ); + } + } + + #[test] + fn v4_detection_prefers_config_model_type() { + // config.json `model_type = "deepseek_v4"` wins regardless of what + // the operator calls the model via --served-model-name. + let v4 = Some("deepseek_v4".to_string()); + for display in ["dsflash", "my-pet-model", "llama-3-8b", ""] { + assert!( + is_deepseek_v4(&v4, display), + "config says deepseek_v4, display {display:?} — expected V4", + ); + } + + // A concrete non-V4 config.json suppresses the display-name fallback. + // Even if the operator names the served model "deepseek-v4", a model + // with `model_type = "llama"` is NOT DeepSeek-V4. + let llama = Some("llama".to_string()); + for display in ["deepseek-v4", "deepseek-v4-flash", "anything"] { + assert!( + !is_deepseek_v4(&llama, display), + "config says llama, display {display:?} — expected NOT V4", + ); + } + + // No config.json — fall back to display-name match. + assert!(is_deepseek_v4(&None, "deepseek-v4-flash")); + assert!(!is_deepseek_v4(&None, "dsflash")); + + // A config.json with `"model_type": ""` is treated as "no signal" at + // the call site (normalized to None before is_deepseek_v4 is called), + // so the display-name fallback still runs — pin that contract. + let empty: Option = None; + assert!(is_deepseek_v4(&empty, "deepseek-v4-flash")); + assert!(!is_deepseek_v4(&empty, "dsflash")); + } + + #[test] + fn v3_2_detection_prefers_config_model_type() { + // config says deepseek_v3_2, any non-"exp" display name triggers. + let v3_2 = Some("deepseek_v3_2".to_string()); + assert!(is_deepseek_v3_2_non_exp(&v3_2, "whatever")); + assert!(is_deepseek_v3_2_non_exp(&v3_2, "deepseek-v3.2")); + // V3.2-Exp is a separate model family; suppress even via config. + assert!(!is_deepseek_v3_2_non_exp(&v3_2, "deepseek-v3.2-exp")); + + // Other config types lose regardless of display name. + let other = Some("deepseek_v4".to_string()); + assert!(!is_deepseek_v3_2_non_exp(&other, "deepseek-v3.2")); + + // No config — fall back to the original display-name heuristic. + assert!(is_deepseek_v3_2_non_exp(&None, "deepseek-v3.2-pro")); + assert!(!is_deepseek_v3_2_non_exp(&None, "deepseek-v3.2-exp")); + assert!(!is_deepseek_v3_2_non_exp(&None, "deepseek-v4")); + } +} diff --git a/lib/llm/tests/data/deepseek-v4/test_input_1.json b/lib/llm/tests/data/deepseek-v4/test_input_1.json new file mode 100644 index 000000000000..de0f6b8b2966 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_input_1.json @@ -0,0 +1 @@ +{"tools":[{"type":"function","function":{"name":"get_weather","description":"Get the weather for a specific location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city name"},"unit":{"type":"string","enum":["celsius","fahrenheit"],"description":"Temperature unit"}},"required":["location"]}}},{"type":"function","function":{"name":"search","description":"Search the web for information","parameters":{"type":"object","properties":{"query":{"type":"string","description":"Search query"},"num_results":{"type":"integer","description":"Number of results to return"}},"required":["query"]}}}],"messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What's the weather in Beijing?"},{"role":"assistant","reasoning_content":"The user wants to know the weather in Beijing. I should use the get_weather tool.","tool_calls":[{"id":"call_001","type":"function","function":{"name":"get_weather","arguments":"{\"location\": \"Beijing\", \"unit\": \"celsius\"}"}}]},{"role":"tool","tool_call_id":"call_001","content":"{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}"},{"role":"assistant","reasoning_content":"Got the weather data. Let me format a nice response.","content":"The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity."}]} diff --git a/lib/llm/tests/data/deepseek-v4/test_input_2.json b/lib/llm/tests/data/deepseek-v4/test_input_2.json new file mode 100644 index 000000000000..81a0a589a717 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_input_2.json @@ -0,0 +1 @@ +[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello"},{"role":"assistant","reasoning_content":"The user said hello, I should greet back.","content":"Hi there! How can I help you?"},{"role":"user","content":"What is the capital of France?"},{"role":"assistant","reasoning_content":"The user asks about the capital of France. It is Paris.","content":"The capital of France is Paris."}] diff --git a/lib/llm/tests/data/deepseek-v4/test_input_3.json b/lib/llm/tests/data/deepseek-v4/test_input_3.json new file mode 100644 index 000000000000..3468700632b4 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_input_3.json @@ -0,0 +1 @@ +[{"role":"system","content":"该助手为DeepSeek，由深度求索公司创造。"},{"role":"latest_reminder","content":"2026-02-21,星期六,广州,App,中文"},{"role":"developer","content":"小柴胡冲剂和布洛芬能一起吃吗？\n\nCITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】","tools":[{"type":"function","function":{"name":"search","description":"Web search. Split multiple queries with '||'.","parameters":{"type":"object","properties":{"queries":{"type":"string","description":"query1||query2"}},"required":["queries"],"additionalProperties":false,"$schema":"http://json-schema.org/draft-07/schema#"}}},{"type":"function","function":{"name":"open","description":"Batch open IDs (format 【{id}†...】) or URLs.","parameters":{"type":"object","properties":{"open_list":{"type":"array","items":{"type":"object","properties":{"id":{"description":"ID or URL","anyOf":[{"type":"integer"},{"type":"string"}],"default":-1},"cursor":{"type":"integer","description":"","default":-1},"loc":{"type":"integer","description":"Start line","default":-1},"num_lines":{"type":"integer","description":"","default":-1},"view_source":{"type":"boolean","description":"","default":false}},"additionalProperties":false},"description":""}},"required":["open_list"],"additionalProperties":false,"$schema":"http://json-schema.org/draft-07/schema#"}}},{"type":"function","function":{"name":"find","description":"Find exact text pattern in pages.","parameters":{"type":"object","properties":{"find_list":{"type":"array","items":{"type":"object","properties":{"pattern":{"type":"string","description":""},"cursor":{"type":"integer","description":"","default":-1}},"required":["pattern"],"additionalProperties":false},"description":""}},"required":["find_list"],"additionalProperties":false,"$schema":"http://json-schema.org/draft-07/schema#"}}}]},{"role":"assistant","content":"","reasoning_content":"用户想知道小柴胡冲剂和布洛芬能否一起服用。","tool_calls":[{"type":"function","function":{"name":"search","arguments":"{\"queries\": \"小柴胡冲剂 布洛芬 相互作用 一起吃\"}"}}]},{"role":"tool","content":"[0]"},{"role":"assistant","content":"请及时就医。","reasoning_content":"现在开始组织回答。","tool_calls":[]}] diff --git a/lib/llm/tests/data/deepseek-v4/test_input_4.json b/lib/llm/tests/data/deepseek-v4/test_input_4.json new file mode 100644 index 000000000000..34feb3f2d896 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_input_4.json @@ -0,0 +1 @@ +[{"role":"system","content":"该助手为DeepSeek-V3，由深度求索公司创造。\n今天是2025年10月17日，星期五。"},{"role":"latest_reminder","content":"2024-11-15,上海市,App,中文"},{"role":"user","content":"热海大滚锅是世界著名温泉吗"},{"role":"assistant","content":"关于热海大滚锅是否是世界著名温泉，可以这样总结：它在**中国乃至全球的地热奇观中占有重要地位**，但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**，而非普遍的温泉疗养体验。\n\n为了让你快速了解，我整理了一个简要的表格：\n\n| 维度 | 热海大滚锅的地位与特点 |\n| :--- | :--- |\n| **地质奇观** | **世界罕见**的高温地热系统，泉眼多、水温高、形态多样。 |\n| **国内声誉** | **中国三大地热区之一**，**国家5A级旅游景区**，自古闻名（徐霞客曾游历并记载）。 |\n| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一，但此说法流传不广，其国际声誉更多建立在地质独特性上。 |\n| **核心体验** | **观赏地热奇观**（如97℃沸腾的“大滚锅”）、**体验温泉煮鸡蛋**。 |\n\n### 💡 游玩攻略与温馨提示\n\n如果你计划前往热海大滚锅，这里有一些实用信息供你参考：\n\n- **门票与开放时间**：\n - **门票**：景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐，价格会更高，例如约**288元**。\n - **开放时间**：景区一般**08:00-18:00**开放，但具体时间可能变动，建议提前核实。\n\n- **特色体验**：\n - **温泉煮鸡蛋**：这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋（约5-8元/串），然后到“大滚锅”旁的指定区域蒸煮，几分钟便可熟食，趣味十足。\n - **金汤足浴**：可以直接用从“大滚锅”流出的温泉水泡脚，缓解旅途疲劳。\n\n- **注意事项**：\n - **安全第一**：“大滚锅”水温极高，务必遵守游览规则，在指定区域内观赏，切勿随意触碰泉水。\n - **规划行程**：建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路，出入口有观光车接送。\n\n希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问，我很乐意提供进一步的信息。","mask":1},{"role":"user","content":"世界著名温泉有哪些","task":"action"},{"role":"assistant","content":"Search"}] diff --git a/lib/llm/tests/data/deepseek-v4/test_output_1.txt b/lib/llm/tests/data/deepseek-v4/test_output_1.txt new file mode 100644 index 000000000000..7e3c9bd5a394 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_output_1.txt @@ -0,0 +1,36 @@ +<｜begin▁of▁sentence｜>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following: + +<｜DSML｜tool_calls> +<｜DSML｜invoke name="$TOOL_NAME"> +<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<｜DSML｜invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}} +{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<｜User｜>What's the weather in Beijing?<｜Assistant｜>The user wants to know the weather in Beijing. I should use the get_weather tool. + +<｜DSML｜tool_calls> +<｜DSML｜invoke name="get_weather"> +<｜DSML｜parameter name="location" string="true">Beijing +<｜DSML｜parameter name="unit" string="true">celsius + +<｜end▁of▁sentence｜><｜User｜>{"temperature": 22, "condition": "sunny", "humidity": 45}<｜Assistant｜>Got the weather data. Let me format a nice response.The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/lib/llm/tests/data/deepseek-v4/test_output_2.txt b/lib/llm/tests/data/deepseek-v4/test_output_2.txt new file mode 100644 index 000000000000..fc397ef54972 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_output_2.txt @@ -0,0 +1 @@ +<｜begin▁of▁sentence｜>You are a helpful assistant.<｜User｜>Hello<｜Assistant｜>Hi there! How can I help you?<｜end▁of▁sentence｜><｜User｜>What is the capital of France?<｜Assistant｜>The user asks about the capital of France. It is Paris.The capital of France is Paris.<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/lib/llm/tests/data/deepseek-v4/test_output_3.txt b/lib/llm/tests/data/deepseek-v4/test_output_3.txt new file mode 100644 index 000000000000..edee563300d4 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_output_3.txt @@ -0,0 +1,38 @@ +<｜begin▁of▁sentence｜>该助手为DeepSeek，由深度求索公司创造。<｜latest_reminder｜>2026-02-21,星期六,广州,App,中文<｜User｜>小柴胡冲剂和布洛芬能一起吃吗？ + +CITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】 + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following: + +<｜DSML｜tool_calls> +<｜DSML｜invoke name="$TOOL_NAME"> +<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<｜DSML｜invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "search", "description": "Web search. Split multiple queries with '||'.", "parameters": {"type": "object", "properties": {"queries": {"type": "string", "description": "query1||query2"}}, "required": ["queries"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}} +{"name": "open", "description": "Batch open IDs (format 【{id}†...】) or URLs.", "parameters": {"type": "object", "properties": {"open_list": {"type": "array", "items": {"type": "object", "properties": {"id": {"description": "ID or URL", "anyOf": [{"type": "integer"}, {"type": "string"}], "default": -1}, "cursor": {"type": "integer", "description": "", "default": -1}, "loc": {"type": "integer", "description": "Start line", "default": -1}, "num_lines": {"type": "integer", "description": "", "default": -1}, "view_source": {"type": "boolean", "description": "", "default": false}}, "additionalProperties": false}, "description": ""}}, "required": ["open_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}} +{"name": "find", "description": "Find exact text pattern in pages.", "parameters": {"type": "object", "properties": {"find_list": {"type": "array", "items": {"type": "object", "properties": {"pattern": {"type": "string", "description": ""}, "cursor": {"type": "integer", "description": "", "default": -1}}, "required": ["pattern"], "additionalProperties": false}, "description": ""}}, "required": ["find_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<｜Assistant｜>用户想知道小柴胡冲剂和布洛芬能否一起服用。 + +<｜DSML｜tool_calls> +<｜DSML｜invoke name="search"> +<｜DSML｜parameter name="queries" string="true">小柴胡冲剂 布洛芬 相互作用 一起吃 + +<｜end▁of▁sentence｜><｜User｜>[0]<｜Assistant｜>现在开始组织回答。请及时就医。<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/lib/llm/tests/data/deepseek-v4/test_output_4.txt b/lib/llm/tests/data/deepseek-v4/test_output_4.txt new file mode 100644 index 000000000000..d30bd5d06cf3 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_output_4.txt @@ -0,0 +1,29 @@ +<｜begin▁of▁sentence｜>该助手为DeepSeek-V3，由深度求索公司创造。 +今天是2025年10月17日，星期五。<｜latest_reminder｜>2024-11-15,上海市,App,中文<｜User｜>热海大滚锅是世界著名温泉吗<｜Assistant｜>关于热海大滚锅是否是世界著名温泉，可以这样总结：它在**中国乃至全球的地热奇观中占有重要地位**，但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**，而非普遍的温泉疗养体验。 + +为了让你快速了解，我整理了一个简要的表格： + +| 维度 | 热海大滚锅的地位与特点 | +| :--- | :--- | +| **地质奇观** | **世界罕见**的高温地热系统，泉眼多、水温高、形态多样。 | +| **国内声誉** | **中国三大地热区之一**，**国家5A级旅游景区**，自古闻名（徐霞客曾游历并记载）。 | +| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一，但此说法流传不广，其国际声誉更多建立在地质独特性上。 | +| **核心体验** | **观赏地热奇观**（如97℃沸腾的“大滚锅”）、**体验温泉煮鸡蛋**。 | + +### 💡 游玩攻略与温馨提示 + +如果你计划前往热海大滚锅，这里有一些实用信息供你参考： + +- **门票与开放时间**： + - **门票**：景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐，价格会更高，例如约**288元**。 + - **开放时间**：景区一般**08:00-18:00**开放，但具体时间可能变动，建议提前核实。 + +- **特色体验**： + - **温泉煮鸡蛋**：这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋（约5-8元/串），然后到“大滚锅”旁的指定区域蒸煮，几分钟便可熟食，趣味十足。 + - **金汤足浴**：可以直接用从“大滚锅”流出的温泉水泡脚，缓解旅途疲劳。 + +- **注意事项**： + - **安全第一**：“大滚锅”水温极高，务必遵守游览规则，在指定区域内观赏，切勿随意触碰泉水。 + - **规划行程**：建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路，出入口有观光车接送。 + +希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问，我很乐意提供进一步的信息。<｜end▁of▁sentence｜><｜User｜>世界著名温泉有哪些<｜Assistant｜><｜action｜>Search<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json new file mode 100644 index 000000000000..087e58bb4a20 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json @@ -0,0 +1,15 @@ +{ + "request_id": "deepseek-v4-content-before-tool-test", + "expected_output": {"normal_content": "Let me check the forecast for Tokyo right now.", "reasoning_content": "The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{\"location\": \"Tokyo\", \"unit\": \"celsius\"}"}}]}, + "input_stream": [ + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather.","role":"assistant","reasoning_content":"The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"Let me check the forecast for Tokyo right now.","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"get_weather\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"location\" string=\"true\">Tokyo\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"unit\" string=\"true\">celsius\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json new file mode 100644 index 000000000000..322ec9f2af27 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json @@ -0,0 +1,195 @@ +{ + "request_id": "deepseek-v4-fragmented-tokens-test", + "expected_output": {"normal_content": "", "reasoning_content": "Break tokens apart aggressively.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "ping", "arguments": "{\"host\": \"example.com\"}"}}]}, + "input_stream": [ + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"B","role":"assistant","reasoning_content":"B"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant","reasoning_content":"r"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant","reasoning_content":"e"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant","reasoning_content":"a"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"k","role":"assistant","reasoning_content":"k"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant","reasoning_content":" "}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant","reasoning_content":"t"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant","reasoning_content":"o"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"k","role":"assistant","reasoning_content":"k"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant","reasoning_content":"e"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant","reasoning_content":"n"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant","reasoning_content":"s"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant","reasoning_content":" "}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant","reasoning_content":"a"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant","reasoning_content":"p"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant","reasoning_content":"a"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant","reasoning_content":"r"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant","reasoning_content":"t"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant","reasoning_content":" "}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant","reasoning_content":"a"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"g","role":"assistant","reasoning_content":"g"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"g","role":"assistant","reasoning_content":"g"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant","reasoning_content":"r"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant","reasoning_content":"e"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant","reasoning_content":"s"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant","reasoning_content":"s"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant","reasoning_content":"i"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"v","role":"assistant","reasoning_content":"v"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant","reasoning_content":"e"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant","reasoning_content":"l"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"y","role":"assistant","reasoning_content":"y"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":".","role":"assistant","reasoning_content":"."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"_","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"c","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"v","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"k","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"=","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"g","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"=","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"h","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"g","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"=","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"u","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"x","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":".","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"c","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"/","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"/","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"v","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"k","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"/","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"_","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"c","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json new file mode 100644 index 000000000000..4210e1c28490 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json @@ -0,0 +1,17 @@ +{ + "request_id": "deepseek-v4-mixed-param-types-test", + "expected_output": {"normal_content": "", "reasoning_content": "The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "send_notification", "arguments": "{\"recipient\": \"user@example.com\", \"priority\": 3, \"urgent\": true, \"tags\": [\"billing\", \"overdue\"], \"metadata\": {\"ticket\": \"T-42\", \"retries\": 2}}"}}]}, + "input_stream": [ + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters.","role":"assistant","reasoning_content":"The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"send_notification\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"recipient\" string=\"true\">user@example.com\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"priority\" string=\"false\">3\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"urgent\" string=\"false\">true\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"tags\" string=\"false\">[\"billing\", \"overdue\"]\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"metadata\" string=\"false\">{\"ticket\": \"T-42\", \"retries\": 2}\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json new file mode 100644 index 000000000000..0129265540d2 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json @@ -0,0 +1,18 @@ +{ + "request_id": "deepseek-v4-multi-tool-test", + "expected_output": {"normal_content": "", "reasoning_content": "The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_current_weather", "arguments": "{\"location\": \"Beijing\", \"format\": \"celsius\"}"}}, {"id": "call_2", "type": "function", "function": {"name": "get_current_weather", "arguments": "{\"location\": \"Shanghai\", \"format\": \"celsius\"}"}}]}, + "input_stream": [ + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information.","role":"assistant","reasoning_content":"The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"get_current_weather\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"location\" string=\"true\">Beijing\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"get_current_weather\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json new file mode 100644 index 000000000000..df0781e1370e --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json @@ -0,0 +1,10 @@ +{ + "request_id": "deepseek-v4-no-tool-test", + "expected_output": {"normal_content": "Hi! I'm here to help — what would you like to work on today?", "reasoning_content": "User greeted me politely. A short friendly reply is appropriate; no tools needed.", "tool_calls": []}, + "input_stream": [ + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"User greeted me politely. A short friendly reply is appropriate; no tools needed.","role":"assistant","reasoning_content":"User greeted me politely. A short friendly reply is appropriate; no tools needed."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"Hi! I'm here to help — ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"what would you like to work on today?","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"stop"}]}} + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json new file mode 100644 index 000000000000..2d5375f10014 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json @@ -0,0 +1,16 @@ +{ + "request_id": "deepseek-v4-special-chars-test", + "expected_output": {"normal_content": "", "reasoning_content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "save_note", "arguments": "{\"note\": \"He said \\\"hello\\\".\\n\\t'world' `backtick` — 中文测试 — 🚀✨ & .\"}"}}]}, + "input_stream": [ + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.","role":"assistant","reasoning_content":"The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"save_note\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"note\" string=\"true\">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"He said \"hello\".\n\t'world' `backtick` ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"— 中文测试 — 🚀✨ ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":" & .\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json new file mode 100644 index 000000000000..47e94f923073 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json @@ -0,0 +1,14 @@ +{ + "request_id": "deepseek-v4-tool-call-test", + "expected_output": {"normal_content": "", "reasoning_content": "User wants the current weather in Beijing. I'll call get_current_weather with celsius units.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_current_weather", "arguments": "{\"location\": \"Beijing\", \"format\": \"celsius\"}"}}]}, + "input_stream": [ + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"User wants the current weather in Beijing. I'll call get_current_weather with celsius units.","role":"assistant","reasoning_content":"User wants the current weather in Beijing. I'll call get_current_weather with celsius units."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"get_current_weather\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"location\" string=\"true\">Beijing\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} + ] +} diff --git a/lib/llm/tests/deepseek_v4_encoding.rs b/lib/llm/tests/deepseek_v4_encoding.rs new file mode 100644 index 000000000000..6a968e4c4b59 --- /dev/null +++ b/lib/llm/tests/deepseek_v4_encoding.rs @@ -0,0 +1,112 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Tests for DeepSeek V4 encoding against official test data +//! +//! These tests use the official test files from: +//! https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/tree/main/encoding + +use dynamo_llm::preprocessor::prompt::deepseek_v4::{ThinkingMode, encode_messages}; +use serde_json::Value as JsonValue; +use std::fs; +use std::path::PathBuf; + +fn get_test_data_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/deepseek-v4") +} + +/// Load an input fixture. V4 fixtures come in two shapes: +/// 1. `{"tools": [...], "messages": [...]}` — tools injected on first (system) message +/// 2. bare `[...]` — just the messages array +fn load_messages(path: &PathBuf) -> Vec { + let raw: JsonValue = serde_json::from_str( + &fs::read_to_string(path).unwrap_or_else(|_| panic!("Failed to read {:?}", path)), + ) + .unwrap_or_else(|_| panic!("Failed to parse {:?}", path)); + + if let Some(messages) = raw.get("messages").and_then(|m| m.as_array()) { + let mut messages = messages.clone(); + if let Some(tools) = raw.get("tools") + && let Some(first) = messages.get_mut(0) + && let Some(obj) = first.as_object_mut() + { + obj.insert("tools".to_string(), tools.clone()); + } + messages + } else if let Some(arr) = raw.as_array() { + arr.clone() + } else { + panic!("Unexpected input shape in {:?}", path); + } +} + +fn run_official_test(input_file: &str, output_file: &str, thinking_mode: ThinkingMode) { + let test_dir = get_test_data_path(); + let messages = load_messages(&test_dir.join(input_file)); + let expected = fs::read_to_string(test_dir.join(output_file)) + .unwrap_or_else(|_| panic!("Failed to read {}", output_file)); + + let actual = encode_messages(&messages, thinking_mode, true) + .unwrap_or_else(|e| panic!("encode_messages failed for {}: {:?}", input_file, e)); + + let exp = expected.trim_end(); + let act = actual.trim_end(); + + if exp != act { + println!("=== Test: {} ===", input_file); + let exp_lines: Vec<&str> = exp.lines().collect(); + let act_lines: Vec<&str> = act.lines().collect(); + for (i, (el, al)) in exp_lines.iter().zip(act_lines.iter()).enumerate() { + if el != al { + println!("Line {} differs:", i + 1); + println!(" Expected: {:?}", el); + println!(" Actual: {:?}", al); + break; + } + } + if exp_lines.len() != act_lines.len() { + println!( + "\nLine count mismatch: expected {} lines, got {} lines", + exp_lines.len(), + act_lines.len() + ); + } + panic!("Output does not match expected for {}", input_file); + } +} + +/// Case 1 — thinking mode, single tool, tool result round-trip. +#[test] +fn test_official_thinking_with_tools() { + run_official_test( + "test_input_1.json", + "test_output_1.txt", + ThinkingMode::Thinking, + ); +} + +/// Case 2 — thinking mode, no tools, multi-turn (drop_thinking strips earlier reasoning). +#[test] +fn test_official_thinking_no_tools_multiturn() { + run_official_test( + "test_input_2.json", + "test_output_2.txt", + ThinkingMode::Thinking, + ); +} + +/// Case 3 — thinking mode, developer role with tools + latest_reminder + tool result. +#[test] +fn test_official_developer_with_tools_and_reminder() { + run_official_test( + "test_input_3.json", + "test_output_3.txt", + ThinkingMode::Thinking, + ); +} + +/// Case 4 — chat mode, latest_reminder + task="action" + mask preservation. +#[test] +fn test_official_chat_mode_action_task() { + run_official_test("test_input_4.json", "test_output_4.txt", ThinkingMode::Chat); +} diff --git a/lib/llm/tests/test_streaming_tool_parsers.rs b/lib/llm/tests/test_streaming_tool_parsers.rs index ceb3b1a6c8cf..983b59dcf42d 100644 --- a/lib/llm/tests/test_streaming_tool_parsers.rs +++ b/lib/llm/tests/test_streaming_tool_parsers.rs @@ -1106,6 +1106,167 @@ mod tests { ); } + // ---- DeepSeek V4 (DSML format) streaming parser tests ---- + // + // V4 emits tool calls inside a DSML block: + // <｜DSML｜tool_calls> + // <｜DSML｜invoke name="fn"> + // <｜DSML｜parameter name="k" string="true|false">v + // + // + // Fixtures live under tests/data/vllm/deepseek-v4/. + + /// Shared harness for DeepSeek V4 e2e fixtures that end in a tool call. + async fn run_deepseek_v4_tool_call_fixture(file_path: &str) { + let test_data = load_test_data(file_path); + let input_stream = stream::iter(test_data.stream_chunks); + + let output_chunks = parse_response_stream( + input_stream, + true, + true, + Some("deepseek_v4".to_string()), + Some("deepseek_v4".to_string()), + ) + .await; + + assert!(!output_chunks.is_empty(), "Should have output chunks"); + + let aggregated = aggregate_content_from_chunks(&output_chunks); + + assert_eq!( + aggregated.reasoning_content, test_data.expected_reasoning_content, + "Should have extracted reasoning content.", + ); + assert_eq!( + aggregated.normal_content, test_data.expected_normal_content, + "Normal content should match expected value.", + ); + + let expected_has_tool_calls = !test_data.expected_tool_calls.is_empty(); + assert_eq!( + aggregated.has_tool_calls, expected_has_tool_calls, + "Tool calls presence should match expected value" + ); + assert_tool_calls(&aggregated.tool_calls, &test_data.expected_tool_calls); + + assert!( + validate_finish_reason(&output_chunks, FinishReason::ToolCalls), + "finish_reason validation failed for tool call case" + ); + } + + /// Single tool call, thinking mode (direct V4 analog of the V3 tool fixture). + /// `CASE.1` + `CASE.8` + `CASE.9` — single tool call, streaming assembly, paired with reasoning. Also validates `CASE.12` finish_reason=tool_calls. + #[tokio::test] + async fn test_deepseek_v4_e2e_with_tools_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_tool.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// No tool call — thinking + plain body; finish_reason=stop. + /// `CASE.3` + `CASE.10` — no tool call + reasoning only. Also validates `CASE.12` finish_reason=stop. + #[tokio::test] + async fn test_deepseek_v4_e2e_with_no_tools_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_no_tool.json", + DATA_ROOT_PATH + ); + let test_data = load_test_data(&file_path); + let input_stream = stream::iter(test_data.stream_chunks); + + let output_chunks = parse_response_stream( + input_stream, + true, + true, + Some("deepseek_v4".to_string()), + Some("deepseek_v4".to_string()), + ) + .await; + + assert!(!output_chunks.is_empty(), "Should have output chunks"); + + let aggregated = aggregate_content_from_chunks(&output_chunks); + + assert_eq!( + aggregated.reasoning_content, test_data.expected_reasoning_content, + "Should have extracted reasoning content.", + ); + assert_eq!( + aggregated.normal_content, test_data.expected_normal_content, + "Normal content should match expected value.", + ); + assert!(!aggregated.has_tool_calls, "Should not have any tool calls"); + + assert!( + validate_finish_reason(&output_chunks, FinishReason::Stop), + "finish_reason validation failed for non-tool call case" + ); + } + + /// Two parallel tool calls inside one DSML block. + /// `CASE.2` — parallel tool calls in one DSML block. + #[tokio::test] + async fn test_deepseek_v4_e2e_multi_tool_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_multi_tool.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// string="true" vs string="false" — numbers, booleans, arrays, objects must + /// round-trip as their proper JSON types inside arguments. + /// `CASE.7` — complex args (mixed string="true|false" → strings / numbers / bools / arrays / objects round-trip). + #[tokio::test] + async fn test_deepseek_v4_e2e_mixed_param_types_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// Body text emitted before the DSML block — parser must populate both + /// normal_content and tool_calls. + /// `CASE.13` — normal text interleaved before the DSML block. + #[tokio::test] + async fn test_deepseek_v4_e2e_content_before_tool_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// Parameter value containing unicode, emoji, embedded quotes/newlines/tabs, + /// and fragments that look like sentinels but aren't — must not confuse the + /// parser, which anchors only on the exact token. + /// `CASE.7` — Unicode / special characters inside argument values. (`CASE.xml.entities` is N/A for DSML — no entity decoding.) + #[tokio::test] + async fn test_deepseek_v4_e2e_special_chars_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_special_chars.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// Adversarial streaming: every DSML character is its own delta (~200 chunks). + /// Exercises buffer accumulation across chunk boundaries. + /// `CASE.8` — streaming chunk-boundary splits (tokens straddle chunks). + #[tokio::test] + async fn test_deepseek_v4_e2e_fragmented_tokens_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + // ---- Kimi K2 streaming jail reproduction tests ---- // // These reproduce the customer-reported issue (DIS-1765): Kimi K2 agentic diff --git a/lib/parsers/README.md b/lib/parsers/README.md new file mode 100644 index 000000000000..3a6bcac2e62f --- /dev/null +++ b/lib/parsers/README.md @@ -0,0 +1,129 @@ +# dynamo-parsers + +Rust crate for parsing **tool calls** and **reasoning content** out of raw LLM +output. Wire-format-aware, streaming-first, model-family-aware. + +This is the post-model side of Dynamo's chat-completions pipeline: given a +token stream from vLLM or SGLang, extract structured `Vec` + +`reasoning_content` for the client. The pre-model side (prompt formatting) +lives in `lib/llm/src/preprocessor/prompt/`. + +## What's in the crate + +Two top-level modules, each with its own parser registry: + +``` +lib/parsers/ +├── src/ +│ ├── tool_calling/ ← tool-call extraction (17 registered parsers) +│ │ ├── parsers.rs — registry + dispatch (detect_and_parse_tool_call) +│ │ ├── config.rs — per-parser ToolCallConfig +│ │ ├── response.rs — ToolCallResponse shape (wire type) +│ │ ├── dsml/ — DeepSeek V3.2 / V4 DSML grammar +│ │ ├── xml/ — hermes, glm47, kimi_k2, minimax_m2, qwen3_coder +│ │ ├── json/ — deepseek_v3, deepseek_v3_1, nemotron_deci/nano, jamba, mistral, phi4, llama3_json +│ │ ├── harmony/ — OpenAI gpt-oss (Harmony token stream, uses openai_harmony crate) +│ │ └── pythonic/ — Python function-call syntax (some Llama variants) +│ └── reasoning/ ← reasoning-content extraction (14 registered parsers) +│ ├── mod.rs — registry + dispatch +│ ├── base_parser.rs — BasicReasoningParser ( ... ) +│ ├── gpt_oss_parser.rs — Harmony channel parsing +│ ├── granite_parser.rs — Granite-style +│ └── minimax_append_think_parser.rs — MiniMax inline-reasoning +``` + +## How a request flows through the crate + +``` + token stream from engine + │ + ▼ + ┌─────────────────────────────────┐ + │ reasoning parser │ — registered by name via + │ (basic / gpt_oss / ...) │ reasoning::mod.rs get_reasoning_parser_map() + │ │ returns: (reasoning_content, non_reasoning_tail) + └─────────────────────────────────┘ + │ + ▼ (non-reasoning tail) + ┌─────────────────────────────────┐ + │ tool-call parser │ — registered by name via + │ dispatched on parser name │ tool_calling::parsers::get_tool_parser_map() + │ which picks a ParserConfig: │ + │ - Dsml(DsmlParserConfig) │ → try_tool_call_parse_dsml + │ - Json(JsonParserConfig) │ → try_tool_call_parse_json + │ - Xml(XmlParserConfig) │ → try_tool_call_parse_xml + │ - KimiK2(KimiK2ParserConfig)│ → try_tool_call_parse_kimi_k2 + │ - Pythonic / Harmony │ + └─────────────────────────────────┘ + │ + ▼ + Vec + normal_text +``` + +Main public entry points in `tool_calling/parsers.rs`: + +- `detect_and_parse_tool_call(input, parser_name, schema) -> (calls, normal_text)` +- `try_tool_call_parse(input, config) -> (calls, normal_text)` (lower-level, bypasses the registry) +- `detect_tool_call_start(chunk, parser_name)` — streaming: "is this chunk starting a tool-call block?" +- `find_tool_call_end_position(chunk, parser_name)` — streaming: "where does the block end in this chunk?" + +## Parser-family cheat sheet + +When adding a new model, the right parser family is usually one of: + +| Family | Grammar | Shared engine | Examples | +| -- | -- | -- | -- | +| **DSML** | `<｜DSML｜tool_calls>...` with typed `string="true|false"` parameters | `dsml/parser.rs` | DeepSeek V3.2, V4 | +| **XML** | `...` with nested `` or `` | `xml/parser.rs` (generic) or own file for variants | hermes, qwen3_coder, minimax_m2, glm47 (own), kimi_k2 (own, special-token XML) | +| **JSON** | Start sentinel + bare JSON array of `{name, arguments}` | `json/base_json_parser.rs` | deepseek_v3, deepseek_v3_1, nemotron_deci/nano | +| **Harmony** | OpenAI Harmony token stream with `<\|channel\|>`, `<\|message\|>`, `<\|call\|>` | `harmony/harmony_parser.rs` (wraps external `openai_harmony` crate) | gpt-oss-20B / 120B | +| **Pythonic** | `[func_name(arg=value, ...)]` Python function-call syntax | `pythonic/pythonic_parser.rs` | some Llama variants | + +Reasoning parsers: + +| Family | Grammar | Shared engine | Examples | +| -- | -- | -- | -- | +| **Basic (think-tag)** | `...` | `reasoning/base_parser.rs` (BasicReasoningParser) | Qwen3, Nemotron, Kimi K2.5, DeepSeek R1 / V4, GLM-4.5+ | +| **Append-think** | `...` left inline as text, with `` prefix on first chunk | `reasoning/minimax_append_think_parser.rs` | MiniMax M2 | +| **Harmony channel** | Hidden `analysis` channel | `reasoning/gpt_oss_parser.rs` (wraps external `openai_harmony`) | gpt-oss-20B / 120B | +| **Granite** | Custom start/end tokens | `reasoning/granite_parser.rs` | IBM Granite | + +## Adding a new parser + +1. **Pick the family** from the cheat sheet above. If an existing config-driven + family fits, add a `ToolCallConfig::()` constructor in + `tool_calling/config.rs`, register it in `tool_calling/parsers.rs`. Done — + you inherit all the shared parser and tests. + +2. **If the grammar is genuinely new**, add a module under `tool_calling/` and + add a `ParserConfig` variant in `config.rs`. Follow the existing parser + modules for layout. + +3. **For reasoning**, prefer aliasing to `BasicReasoningParser` unless the + grammar truly diverges (append-think, Harmony channels). Most new models + use plain `...` and can share. + +4. **Write tests.** Minimum viable set is in [`TESTING.md`](./TESTING.md) (T1–T20 + taxonomy). At minimum: T1/T2/T3 for correctness, T5 for truncation + behavior, T8/T9 for streaming, T14 for interleaved text. `N/A` categories + should be explicitly called out in a comment rather than silently skipped. + +## Related docs + +- [`TESTING.md`](./TESTING.md) — corner-case taxonomy (T1–T20). What every + parser should be tested against, what's N/A per family, what's a universal + gap today. +- `lib/llm/tests/data/` — captured streaming fixtures per (engine × model) + that feed `test_streaming_tool_parsers.rs`. The replay side of the testing + story. + +## Integration with the rest of Dynamo + +- `lib/llm/src/preprocessor/prompt/` — pre-model side. Writes the prompts + that (eventually) come back and get parsed here. +- `lib/llm/src/preprocessor.rs` — top-level request/response pipeline. + Decides whether to run the reasoning parser based on + `is_reasoning_disabled_by_request`, then hands the reasoning-stripped + tail to the tool-call parser. +- `components/src/dynamo/frontend/` — Python frontend that surfaces parsed + output as OpenAI-compatible SSE chunks to the client. diff --git a/lib/parsers/TESTING.md b/lib/parsers/TESTING.md new file mode 100644 index 000000000000..a977196da8ac --- /dev/null +++ b/lib/parsers/TESTING.md @@ -0,0 +1,317 @@ +# Tool-Call / Reasoning Parser Corner Cases + +Reference taxonomy for unit testing tool-call and reasoning parsers. Each parser +added under `src/tool_calling/` or `src/reasoning/` should cover the generic +`CASE.` categories; family-specific parsers also cover their respective +`CASE.xml` / `CASE.harmony` categories. `N/A` should be called out +explicitly in the test file rather than silently omitted. + +Category layout: +- **`CASE.1`–`CASE.16`** — **Generic**. Apply to every parser regardless of grammar. +- **`CASE.xml1`–`CASE.xml2`** — XML-family only (hermes, glm47, qwen3_coder, minimax_m2, kimi_k2). +- **`CASE.harmony1`** — Harmony only (gpt-oss). + +Per-model gap tracking lives elsewhere (not in this repo). + +## Quick reference + +### Generic (all parsers) + +- **`CASE.1`** Single tool call — happy path (one complete, well-formed call). Ex: `xml::test_parse_simple_tool_call`. +- **`CASE.2`** Multiple tool calls — sequential or parallel (2+ in one response). Ex: `tool_choice::test_streaming_required_tool_parallel`. +- **`CASE.3`** No tool call (response is text only). Ex: `xml::test_parse_no_tool_calls`. +- **`CASE.4`** Malformed / partial JSON args (truncated, missing close brace, invalid syntax). Ex: `deepseek_v3::test_parse_..._with_invalid_json`, `xml::test_parse_missing_..._closing_tag`. +- **`CASE.5`** Missing end-token recovery (recover calls when `section_end` is absent due to max_tokens / EOS). Ex: `kimi_k2::test_parse_malformed_no_section_end`. +- **`CASE.6`** Empty args (`arguments={}` / no-arg call). Ex: `kimi_k2::test_parse_no_arg_call`. +- **`CASE.7`** Complex arg types (nested objects, arrays, bool, number, Unicode / newlines in values). Ex: `kimi_k2::test_parse_complex_json_arguments`. +- **`CASE.8`** Streaming — token-by-token assembly + chunk-boundary splits. Ex: `basic::test_buffer_state_persistence_across_calls`, `basic::test_partial_token_matching_closing_tag`, `basic::test_kimi_k2_one_shot_split`. +- **`CASE.9`** Paired reasoning + tool in same response. Ex: `test_reasoning_parser::test_nemotron_with_reasoning_and_tool_calls`. +- **`CASE.10`** Reasoning only (think tags, no tool call). Ex: `basic::test_detect_and_parse_reasoning_reasoning`. +- **`CASE.11`** `tool_choice` = auto / required / named / none. Ex: `tool_choice::test_named_tool_choice_parses_json` (**hermes only today**). +- **`CASE.12`** `finish_reason` semantics (`stop` / `tool_calls` / `length` mapping). Ex: `tool_choice_finish_reasons::*` (hermes only); `test_streaming_tool_parsers::test_qwen_finish_reason_length_vllm`. +- **`CASE.13`** Normal text interleaved with tool calls. Ex: `xml::test_parse_with_normal_text`. +- **`CASE.14`** Empty content / empty `tool_calls` array / null response. Ex: `parallel_tool_call_integration::test_empty_tool_calls`. +- **`CASE.15`** Duplicate tool calls (same name twice). No test anywhere in the repo; universal gap. +- **`CASE.16`** Regression for a specific customer bug (ticket ID referenced in test name or body). Ex: `kimi_k2::test_parse_malformed_no_section_end`. + +### XML-family (`CASE.xml*`) + +- **`CASE.xml1`** XML entity / HTML unescape handling (`<`, `&`, `"` in parameter values). Ex: `xml::test_html_unescape`, `glm47::test_xml_entity_decoding`. +- **`CASE.xml2`** Schema-aware type coercion (string → number/bool/array based on declared parameter schema). Ex: `xml::test_schema_aware_type_conversion`, `glm47::test_type_coercion_*`. + +### Harmony (`CASE.harmony*`) + +- **`CASE.harmony1`** Channel / recipient parsing (analysis / commentary / final channels). Ex: `harmony_parser::test_parse_tool_calls_harmony_*`. + +### Universal gaps (no test anywhere, not promoted to numbered categories) + +- Unicode in function names (non-ASCII tool names, emoji). +- Numeric overflow in args (very large int / float outside JSON spec range). +- Empty function name (`"name": ""`). +- Concurrent parallel requests (process-level contention during parse). +- Guided-decoding ↔ tool-call interaction (constrained generation emits malformed args). +- Extremely long output (≥10 KB tool-call JSON in a single call). +- Mid-stream error injection / interruption (worker kill, network drop mid-parse). +- Schema arg-count mismatch (model emits extra or missing args vs declared schema). + +--- + +## `CASE.1` — Single tool call, happy path + +One complete, well-formed call in the response. + +- Applies to every tool-call parser. +- Baseline correctness check. If `CASE.1` fails, nothing else below matters. +- Example: `dsml/parser.rs::test_parse_single_tool_call_string_param`. + +## `CASE.2` — Multiple tool calls (sequential or parallel) + +Two or more calls in one response, in the same block or back-to-back. + +- Applies to every tool-call parser. +- Some grammars emit parallel calls in one block (DSML, XML); others emit + sequential top-level sentinels (JSON dialects). Either way, extract all. +- Example: `dsml/parser.rs::test_parse_multiple_tool_calls`, + `tool_choice::test_streaming_required_tool_parallel`. + +## `CASE.3` — No tool call + +Response is plain text, no tool-call grammar present. + +- Applies to every tool-call parser. +- Must return empty `Vec` and the input as `normal_text`. Zero false + positives. +- Example: `dsml/parser.rs::test_parse_no_tool_calls`. + +## `CASE.4` — Malformed / partial JSON args + +Truncated JSON, missing close brace, invalid syntax inside the arguments +payload. + +- Applies to every tool-call parser. For parsers whose grammar never embeds + JSON (none today — all top-N families embed JSON somewhere), mark explicit + `N/A`. +- Behavior must be documented: either graceful fallback to string (DSML's + current behavior via `serde_json::from_str(...).unwrap_or_else(|_| String(...))`) + or explicit error. Silent drop is the failure mode. +- Example: `dsml/parser.rs::test_parse_deepseek_v4_malformed_json_value_falls_back_to_string`, + `deepseek_v3_parser.rs::test_parse_tool_calls_deepseek_v3_with_invalid_json`. + +## `CASE.5` — Missing end-token recovery + +The model's response is truncated before the closing fence arrives +(`<|tool_calls_section_end|>` for Kimi, `` for DeepSeek +DSML, etc.) — typically because the engine hit `max_tokens` or the model +emitted EOS mid-generation. + +- Applies to every tool-call parser with paired start/end fences. +- Customer-facing bug class: silent drop of the in-flight call looks like a + successful HTTP 200 with no tool_calls and no error. +- Two acceptable resolutions: (a) recover completed invokes even without the + outer close fence (Kimi K2 does this post-fix), or (b) return an explicit + error. Either way, pin the behavior with a test so a future change is + intentional. +- Example (post-recovery): `kimi_k2_parser.rs::test_parse_malformed_no_section_end`. +- Example (behavior-pinning, pre-recovery): `dsml/parser.rs::test_parse_deepseek_v4_missing_end_token`. + +## `CASE.6` — Empty args + +Tool call with `arguments={}`, or a no-parameter invoke. + +- Applies to every tool-call parser. +- Must still return the call — empty args is a valid call, not a missing one. +- Example: `kimi_k2_parser.rs::test_parse_no_arg_call`, + `dsml/parser.rs::test_parse_deepseek_v4_no_parameters`. + +## `CASE.7` — Complex argument types + +Nested objects, arrays, booleans, numbers, mixed types, Unicode values, and +newlines inside argument values. + +- Applies to every tool-call parser. +- For grammars that carry type hints (DSML's `string="true|false"`), verify + JSON round-tripping. For XML grammars without hints, the type-coercion + half of the test is covered under `CASE.xml2` instead — here just verify + that complex values make it through without truncation or escape bugs. +- Example: `dsml/parser.rs::test_parse_mixed_types_realistic`, + `kimi_k2_parser.rs::test_parse_complex_json_arguments`. + +## `CASE.8` — Streaming + +Chunked input arriving over SSE. Covers two concerns that tend to fail +together: + +1. **Token-by-token assembly** — the parser incrementally reconstructs the + tool-call structure across many small chunks. +2. **Chunk-boundary splits** — start fence, end fence, or parameter name / + value straddles a chunk boundary. Partial-token matching must return + `true` (keep buffering, don't flush as plain text) and complete the + match on the next chunk. + +- Applies to every tool-call parser. Dominant production path. +- Example: `basic::test_buffer_state_persistence_across_calls`, + `basic::test_partial_token_matching_closing_tag`, + `basic::test_kimi_k2_one_shot_split`, + `test_streaming_tool_parsers::test_deepseek_v4_e2e_fragmented_tokens_vllm`. + +## `CASE.9` — Paired reasoning + tool in same response + +Model emits `...` (or analog) followed by a tool call. Both +must be extracted: `reasoning_content` populated AND `tool_calls` populated. + +- Applies to every (tool, reasoning) parser pair. +- Watch for the "unclosed think-tag swallows tool call" bug — if the reasoning + parser is greedy it may eat the tool-call content that follows. +- Example: `test_reasoning_parser::test_nemotron_with_reasoning_and_tool_calls`, + `test_reasoning_parser::test_kimi_k25_with_reasoning_and_tool_calls`. + +## `CASE.10` — Reasoning only + +`...` or analog present, no tool call. Parser must populate +`reasoning_content` and leave `tool_calls` empty. + +- Applies to every reasoning parser. +- Example: `reasoning/base_parser.rs::test_detect_and_parse_reasoning_reasoning`, + `reasoning/mod.rs::test_deepseek_v4_detect_and_parse`. + +## `CASE.11` — `tool_choice` = auto / required / named / none + +Each of the four OpenAI `tool_choice` modes exercised per parser. + +- Applies to every tool-call parser. +- Cross-parser suites at `lib/llm/tests/tool_choice.rs` / + `parallel_tool_call_integration.rs` / `tool_choice_finish_reasons.rs` + run `hermes` only today. Adding a new parser requires parametrizing those + suites or adding a per-parser equivalent. +- Universal gap across most parsers in the repo as of 2026-04. +- Example: `tool_choice::test_named_tool_choice_parses_json`, + `tool_choice::test_required_tool_choice_parses_json_array`. + +## `CASE.12` — `finish_reason` semantics + +`stop` vs `tool_calls` vs `length` mapping, in both streaming and +non-streaming paths. + +- Applies to every tool-call parser. +- When a tool call lands, `finish_reason` must become `tool_calls`. When + `max_tokens` truncates mid-stream, `length` must propagate — this is + often the signal that should trigger `CASE.5` recovery on the parser side. +- Example: `tool_choice_finish_reasons::test_named_tool_choice_normal_stop_becomes_tool_calls`, + `test_streaming_tool_parsers::test_qwen_finish_reason_length_vllm`. + +## `CASE.13` — Normal text interleaved with tool calls + +Model emits narration text before / after / between tool-call blocks. Parser +must split content correctly: text → `normal_content`, calls → `tool_calls`. + +- Applies to every tool-call parser. +- Example: `dsml/parser.rs::test_parse_with_normal_text`, + `test_streaming_tool_parsers.rs::test_deepseek_v4_e2e_content_before_tool_vllm`. + +## `CASE.14` — Empty content / empty `tool_calls` array / null response + +Engine emits a chunk with `delta.content = ""`, or a final response with +`tool_calls: []`, or `null` values inside arguments. + +- Applies to every tool-call parser. +- Null-value handling inside parameters is parser-level (`parse_parameters` + in DSML handles it via `serde_json::Value::Null`). Empty-choices / + empty-stream handling is typically at the e2e integration layer. +- Example: `dsml/parser.rs::test_parse_null_parameter`, + `parallel_tool_call_integration::test_empty_tool_calls`. + +## `CASE.15` — Duplicate tool calls (same name twice) + +Two calls to the same function name in one response, possibly with the same +arguments. + +- Applies to every tool-call parser. +- **Zero coverage across the entire repo as of 2026-04.** Universal gap. +- Expected behavior: both calls must appear in `tool_calls` with distinct + IDs. (The runtime / client is responsible for deciding whether duplicate + invocation is intended.) + +## `CASE.16` — Regression for a specific customer bug + +Test named after (or containing) a ticket reference, pinning the fix for +a customer-reported failure. + +- Applies per-incident. Not a category every parser needs to cover in + advance; populated as bugs are reported and fixed. +- Existing example: `kimi_k2_parser.rs::test_parse_malformed_no_section_end`. + +--- + +## `CASE.xml1` — XML entity / HTML unescape handling + +Parameter values contain XML-encoded entities (`<`, `&`, `"`, +`'`, numeric entities like `&`) that must be decoded before the +value is surfaced to the client. + +- Applies only to XML-family tool-call parsers: `hermes`, `glm47`, + `qwen3_coder`, `minimax_m2`, `kimi_k2` (despite its special-token outer + fence, the inner parameter payload is XML-ish). +- **N/A for DSML** — the `string="true|false"` attribute tells the parser + whether to JSON-decode or pass through verbatim; no entity decoding pass. +- **N/A for JSON-family and Harmony** — JSON has its own escape semantics + handled by `serde_json`. +- Example: `xml/parser.rs::test_html_unescape`, + `glm47_parser.rs::test_xml_entity_decoding`. + +## `CASE.xml2` — Schema-aware type coercion + +Parser uses the declared tool schema to coerce string args to +number / bool / array based on the declared parameter type. + +- Applies only to XML-family parsers without explicit type annotations in + the wire format. `xml/parser.rs`, `glm47_parser.rs` do this. +- **N/A for DSML** — the `string="true|false"` attribute carries the type + intent per parameter, so no schema lookup is needed. +- **N/A for JSON-family** — JSON has native types. +- **N/A for Harmony** — payload is JSON inside the channel envelope. +- Example: `xml/parser.rs::test_schema_aware_type_conversion`, + `glm47_parser.rs::test_type_coercion_array_comma_separated`. + +--- + +## `CASE.harmony1` — Channel / recipient parsing + +OpenAI Harmony's token stream carries channel metadata +(`<|channel|>analysis|commentary|final<|message|>`) and recipient targets +(`to=functions.foo`). Parser must route the `commentary` channel content +into tool-call extraction while surfacing `analysis` as reasoning and +`final` as the user-visible output. + +- **Harmony only.** N/A for every other family. +- Example: `harmony/harmony_parser.rs::test_parse_tool_calls_harmony_with_multi_args`, + `harmony/harmony_parser.rs::test_parse_tool_calls_harmony_with_normal_text`. + +--- + +## Applicability summary + +| Category block | Parsers | Notes | +| -- | -- | -- | +| `CASE.1`–`CASE.16` (generic) | All | Required contract for every parser | +| `CASE.xml1`–`CASE.xml2` | XML-family only | Entity decoding + schema-aware coercion | +| `CASE.harmony1` | Harmony only | Channel routing | + +## Adding a new parser: what you must include + +Minimum viable set for a new tool-call parser: + +1. `CASE.1`, `CASE.2`, `CASE.3` — baseline correctness. +2. `CASE.4` or explicit N/A justification — handle or refuse malformed input. +3. `CASE.5` — pin behavior when the outer fence is missing. Silent drop is a + regression waiting to happen. +4. `CASE.6`, `CASE.7` — empty and complex args. +5. `CASE.8` — streaming. Essentially non-negotiable for any parser that sits + behind a streaming frontend. +6. `CASE.13` — interleaved text. +7. `CASE.15` — document whether duplicate calls are supported. Flat gap + today; landing a test with the parser establishes the contract. +8. Family-specific categories where applicable: `CASE.xml1` / `CASE.xml2` + for XML grammars, `CASE.harmony1` for Harmony. + +For reasoning parsers, replace `CASE.4` / `CASE.5` / `CASE.8`-assembly with +`CASE.8`-partial-close-tag and `CASE.10` (reasoning-only). diff --git a/lib/parsers/src/reasoning/mod.rs b/lib/parsers/src/reasoning/mod.rs index 3a8a568c2477..5f259bc80f74 100644 --- a/lib/parsers/src/reasoning/mod.rs +++ b/lib/parsers/src/reasoning/mod.rs @@ -29,6 +29,22 @@ fn get_reasoning_parser_map() -> &'static HashMap<&'static str, ReasoningParserT map.insert("basic", ReasoningParserType::Basic); map.insert("gpt_oss", ReasoningParserType::GptOss); map.insert("qwen3", ReasoningParserType::Qwen); + // DeepSeek-V4 uses the same `` / `` delimiters as Qwen + // (confirmed against deepseek-ai/DeepSeek-V4-Pro's encoding_dsv4.py) + // so it delegates to the same `BasicReasoningParser` config today. We + // still route through a dedicated `DeepSeekV4` variant rather than + // hard-aliasing to `Qwen` so future divergence (different special + // tokens, max-thinking mode, etc.) has a place to land without rippling + // through Qwen's own config. + // + // The three name aliases exist because callers set this via + // `--dyn-reasoning-parser` / `--reasoning-parser` with whatever string + // the HF model / vLLM recipe / chat-template author picked. We accept + // all three separator conventions (snake / kebab / concat) rather than + // force a single canonical form on users. + map.insert("deepseek_v4", ReasoningParserType::DeepSeekV4); + map.insert("deepseek-v4", ReasoningParserType::DeepSeekV4); + map.insert("deepseekv4", ReasoningParserType::DeepSeekV4); map.insert("nemotron_deci", ReasoningParserType::NemotronDeci); map.insert("kimi", ReasoningParserType::Kimi); map.insert("kimi_k25", ReasoningParserType::KimiK25); @@ -110,6 +126,14 @@ pub enum ReasoningParserType { Basic, GptOss, Qwen, + /// DeepSeek-V4-Pro / V4-Flash. Currently uses the same `` / + /// `` `BasicReasoningParser` config as Qwen (V4 never appends + /// `` in the completion — the chat template always pre-injects it, + /// so the parser starts via `set_in_reasoning(true)` rather than + /// `force_reasoning`). A dedicated variant keeps future V4-specific + /// divergence (different delimiters, thinking-effort modes) from leaking + /// into Qwen's behavior. + DeepSeekV4, NemotronDeci, Kimi, KimiK25, @@ -161,6 +185,12 @@ impl ReasoningParserType { ReasoningParserType::Qwen => ReasoningParserWrapper { parser: Box::new(basic_parser), }, + // Same `` / `` config as Qwen today; kept as a + // distinct variant so V4-specific divergence has somewhere to land. + // See `ReasoningParserType::DeepSeekV4` docstring for rationale. + ReasoningParserType::DeepSeekV4 => ReasoningParserWrapper { + parser: Box::new(basic_parser), + }, ReasoningParserType::NemotronDeci => ReasoningParserWrapper { parser: Box::new(basic_parser), }, @@ -246,6 +276,9 @@ mod tests { "basic", "gpt_oss", "qwen3", + "deepseek_v4", + "deepseek-v4", + "deepseekv4", "nemotron_deci", "kimi", "kimi_k25", @@ -261,6 +294,45 @@ mod tests { assert!(parsers.contains(&parser)); } } + /// `CASE.10` — reasoning-only (V4 ``/``). + + #[test] + fn test_deepseek_v4_detect_and_parse() { + for parser_name in ["deepseek_v4", "deepseek-v4", "deepseekv4"] { + let mut parser = ReasoningParserType::get_reasoning_parser_from_name(parser_name); + let result = parser.detect_and_parse_reasoning("thinkinganswer", &[]); + assert_eq!(result.reasoning_text, "thinking"); + assert_eq!(result.normal_text, "answer"); + } + } + /// `CASE.3` / `CASE.10` — no reasoning tags ⇒ no `reasoning_content`. + + #[test] + fn test_deepseek_v4_no_forced_reasoning_without_tags() { + let mut parser = ReasoningParserType::get_reasoning_parser_from_name("deepseek_v4"); + let result = parser.detect_and_parse_reasoning("answer only", &[]); + assert_eq!(result.reasoning_text, ""); + assert_eq!(result.normal_text, "answer only"); + } + /// `CASE.8` — streaming reasoning parse (chunked). + + #[test] + fn test_deepseek_v4_streaming() { + let mut parser = ReasoningParserType::get_reasoning_parser_from_name("deepseek_v4"); + + let chunks = ["rea", "sonanswer"]; + let mut reasoning = String::new(); + let mut normal = String::new(); + + for chunk in chunks { + let result = parser.parse_reasoning_streaming_incremental(chunk, &[]); + reasoning.push_str(&result.reasoning_text); + normal.push_str(&result.normal_text); + } + + assert_eq!(reasoning, "reason"); + assert_eq!(normal, "answer"); + } #[test] fn test_kimi_k25_detect_and_parse() { diff --git a/lib/parsers/src/tool_calling/config.rs b/lib/parsers/src/tool_calling/config.rs index c0a5a46448c2..35ab92208f7e 100644 --- a/lib/parsers/src/tool_calling/config.rs +++ b/lib/parsers/src/tool_calling/config.rs @@ -391,6 +391,22 @@ impl ToolCallConfig { } } + pub fn deepseek_v4() -> Self { + // DeepSeek V4 format (DSML): + // <｜DSML｜tool_calls> + // <｜DSML｜invoke name="function_name"> + // <｜DSML｜parameter name="param_name" string="true|false">value + // + // + Self { + parser_config: ParserConfig::Dsml(DsmlParserConfig { + function_calls_start: "<｜DSML｜tool_calls>".to_string(), + function_calls_end: "".to_string(), + ..Default::default() + }), + } + } + pub fn minimax_m2() -> Self { // MiniMax-M2.1 format: // diff --git a/lib/parsers/src/tool_calling/dsml/parser.rs b/lib/parsers/src/tool_calling/dsml/parser.rs index 3f23c9c0ac5f..cec8e75f67db 100644 --- a/lib/parsers/src/tool_calling/dsml/parser.rs +++ b/lib/parsers/src/tool_calling/dsml/parser.rs @@ -5,11 +5,84 @@ // https://huggingface.co/deepseek-ai/DeepSeek-V3.2/tree/main/encoding/encoding_dsv32.py use regex::Regex; +use std::collections::HashMap; +use std::sync::{Arc, OnceLock, RwLock}; use uuid::Uuid; use super::super::config::DsmlParserConfig; use super::super::response::{CalledFunction, ToolCallResponse, ToolCallType}; +/// Compiled regex trio for a given `DsmlParserConfig`. Compiled once and +/// reused across every subsequent parse/stream chunk. +struct DsmlRegexes { + block: Regex, + invoke: Regex, + parameter: Regex, +} + +/// Cache key = the six config strings that drive the three regex patterns. +/// V3.2 and V4 are the only variants in use, so the cache has at most two +/// entries for the lifetime of the process. +type DsmlRegexKey = (String, String, String, String, String, String); + +fn regex_cache() -> &'static RwLock>> { + static CACHE: OnceLock>>> = OnceLock::new(); + CACHE.get_or_init(|| RwLock::new(HashMap::new())) +} + +/// Return the compiled regex trio for `config`, compiling on first use. +/// +/// Each parse call previously recompiled three regexes from `format!`'d +/// patterns that embed the config strings — expensive on streaming hot paths. +/// The cache is keyed on the raw config strings (not the escaped patterns) +/// so distinct configs that happen to escape identically still get distinct +/// entries. +fn get_dsml_regexes(config: &DsmlParserConfig) -> anyhow::Result> { + let key: DsmlRegexKey = ( + config.function_calls_start.clone(), + config.function_calls_end.clone(), + config.invoke_start_prefix.clone(), + config.invoke_end.clone(), + config.parameter_prefix.clone(), + config.parameter_end.clone(), + ); + // Fast path: shared read lock, common after the first parse of each config. + if let Some(regexes) = regex_cache() + .read() + .expect("DSML regex cache read lock poisoned") + .get(&key) + { + return Ok(Arc::clone(regexes)); + } + // Slow path: compile and install. Use `entry` so a concurrent compiler of + // the same key only inserts once (we still compile speculatively, then + // drop the duplicate — cheap on a map of <= 2 keys). + let block = Regex::new(&format!( + r"(?s){}\s*(.*?)\s*{}", + regex::escape(&config.function_calls_start), + regex::escape(&config.function_calls_end), + ))?; + let invoke = Regex::new(&format!( + r#"(?s){}\"([^"]+)\"\s*>(.*?){}"#, + regex::escape(&config.invoke_start_prefix), + regex::escape(&config.invoke_end), + ))?; + let parameter = Regex::new(&format!( + r#"(?s){}\"([^"]+)\"\s+string=\"(true|false)\"\s*>(.*?){}"#, + regex::escape(&config.parameter_prefix), + regex::escape(&config.parameter_end), + ))?; + let regexes = Arc::new(DsmlRegexes { + block, + invoke, + parameter, + }); + let mut cache = regex_cache() + .write() + .expect("DSML regex cache write lock poisoned"); + Ok(Arc::clone(cache.entry(key).or_insert(regexes))) +} + /// DeepSeek V3.2 uses DSML (DeepSeek Markup Language) format for tool calls: /// /// <｜DSML｜function_calls> @@ -97,24 +170,16 @@ fn extract_tool_calls( config: &DsmlParserConfig, ) -> anyhow::Result> { let mut tool_calls = Vec::new(); + let regexes = get_dsml_regexes(config)?; - // Find all function_calls blocks - // Matches: <｜DSML｜function_calls> ... - // Pattern: (?s) = dot matches newlines - // \s*(.*?)\s* = capture content between start/end tags (non-greedy) - let block_pattern = format!( - r"(?s){}\s*(.*?)\s*{}", - regex::escape(&config.function_calls_start), - regex::escape(&config.function_calls_end) - ); - let block_regex = Regex::new(&block_pattern)?; - - for block_match in block_regex.captures_iter(text) { + // Find all function_calls blocks — the block regex captures the content + // between start/end tags (non-greedy, dot-matches-newline). + for block_match in regexes.block.captures_iter(text) { if let Some(block_content) = block_match.get(1) { let block = block_content.as_str(); // Extract individual invokes from this block - let invokes = extract_invokes(block, config)?; + let invokes = extract_invokes(block, ®exes)?; tool_calls.extend(invokes); } } @@ -123,29 +188,18 @@ fn extract_tool_calls( } /// Extract individual invoke blocks from function_calls content -fn extract_invokes( - block: &str, - config: &DsmlParserConfig, -) -> anyhow::Result> { +fn extract_invokes(block: &str, regexes: &DsmlRegexes) -> anyhow::Result> { let mut invokes = Vec::new(); - // Regex to match: <｜DSML｜invoke name="function_name">..content.. - // Note: invoke_start_prefix is "<｜DSML｜invoke name=" (no quotes, we add them in pattern) - let invoke_pattern = format!( - r#"(?s){}\"([^"]+)\"\s*>(.*?){}"#, - regex::escape(&config.invoke_start_prefix), - regex::escape(&config.invoke_end) - ); - let invoke_regex = Regex::new(&invoke_pattern)?; - - for invoke_match in invoke_regex.captures_iter(block) { + // Matches: <｜DSML｜invoke name="function_name">..content.. + for invoke_match in regexes.invoke.captures_iter(block) { if let (Some(name_match), Some(content_match)) = (invoke_match.get(1), invoke_match.get(2)) { let function_name = name_match.as_str().trim().to_string(); let invoke_content = content_match.as_str(); // Parse parameters from invoke content - let parameters = parse_parameters(invoke_content, config)?; + let parameters = parse_parameters(invoke_content, regexes)?; // Create tool call response let arguments_json = serde_json::to_string(¶meters)?; @@ -167,24 +221,12 @@ fn extract_invokes( /// Parse parameters from invoke content fn parse_parameters( content: &str, - config: &DsmlParserConfig, + regexes: &DsmlRegexes, ) -> anyhow::Result> { let mut parameters = serde_json::Map::new(); - // Build pattern with proper escaping - // Match: <｜DSML｜parameter name="param_name" string="true|false">value - // Note: parameter_prefix is "<｜DSML｜parameter name=" (no quotes, we add them in pattern) - let prefix_escaped = regex::escape(&config.parameter_prefix); - let end_escaped = regex::escape(&config.parameter_end); - - let param_pattern = format!( - r#"(?s){}\"([^"]+)\"\s+string=\"(true|false)\"\s*>(.*?){}"#, - prefix_escaped, end_escaped - ); - - let param_regex = Regex::new(¶m_pattern)?; - - for param_match in param_regex.captures_iter(content) { + // Matches: <｜DSML｜parameter name="param_name" string="true|false">value + for param_match in regexes.parameter.captures_iter(content) { if let (Some(name_match), Some(string_match), Some(value_match)) = (param_match.get(1), param_match.get(2), param_match.get(3)) { @@ -224,6 +266,14 @@ mod tests { DsmlParserConfig::default() } + fn get_v4_test_config() -> DsmlParserConfig { + DsmlParserConfig { + function_calls_start: "<｜DSML｜tool_calls>".to_string(), + function_calls_end: "".to_string(), + ..Default::default() + } + } + #[test] fn test_detect_tool_call_start() { let config = get_test_config(); @@ -239,6 +289,74 @@ mod tests { assert!(!detect_tool_call_start_dsml("no tool call here", &config)); } + // ------------------------------------------------------------------- + // DeepSeek V4 coverage (see lib/parsers/TESTING.md for CASE.* taxonomy). + // + // Covered by the V4 tests below (or by a shared DSML generic test): + // - CASE.1 single-call (parsers.rs :: test_deepseek_v4_single_tool_call) + // - CASE.2 multi-calls (test_parse_deepseek_v4_multiple_tool_calls) + // - CASE.3 no-call (shared: test_parse_no_tool_calls) + // - CASE.4 malformed-args (test_parse_deepseek_v4_malformed_json_value_falls_back_to_string, + // test_parse_deepseek_v4_missing_invoke_close_drops_call) + // - CASE.5 missing-end-token (test_parse_deepseek_v4_missing_end_token{,_multiple_calls}) + // — PINNED AS BROKEN: parser drops the call. See TODO below. + // - CASE.6 empty-args (test_parse_deepseek_v4_no_parameters) + // - CASE.7 complex-args (shared: test_parse_mixed_types_realistic, test_parse_nested_object_parameter, + // lib/llm/tests/test_streaming_tool_parsers :: ..._mixed_param_types_vllm, + // ..._special_chars_vllm) + // - CASE.8 streaming (test_detect_tool_call_start_v4, test_find_tool_call_end_position_v4, + // lib/llm/tests/test_streaming_tool_parsers :: ..._fragmented_tokens_vllm) + // - CASE.9 reasoning-plus-tool (lib/llm/tests/test_streaming_tool_parsers :: ..._with_tools_vllm + // — fixtures include ... alongside DSML) + // - CASE.10 reasoning-only (reasoning/mod.rs :: test_deepseek_v4_detect_and_parse etc.) + // - CASE.12 finish-reason (lib/llm/tests/test_streaming_tool_parsers :: ..._with_tools_vllm → + // FinishReason::ToolCalls; ..._with_no_tools_vllm → FinishReason::Stop + // — Length variant NOT covered, see TODO) + // - CASE.13 interleaved-text (test_parse_deepseek_v4_multiple_tool_calls prefix text; + // lib/llm/tests/test_streaming_tool_parsers :: ..._content_before_tool_vllm) + // + // - CASE.xml.* N/A — DSML carries per-parameter string="true|false" type hints, + // so XML entity decoding (CASE.xml.entities) and schema-aware + // coercion (CASE.xml.schema-coercion) don't apply. + // - CASE.harmony.* N/A — Harmony-only. + // + // TODO — not yet covered for V4: + // - CASE.5 Fix mid-stream truncation: parser currently drops all calls when + // is absent (max_tokens / EOS before close). + // Same class as Kimi K2 pre-DIS-1765. Recovery pattern: scan for + // complete <｜DSML｜invoke>... pairs even without + // the outer close fence (see kimi_k2_parser.rs for precedent). + // Pinning tests below capture the current silent-drop behavior; + // flip them when recovery lands. + // - CASE.4 Variants not pinned: missing close tag, + // middle-invoke truncation corrupting subsequent invokes (non-greedy + // regex bleed-through). Same structural class as CASE.5. + // - CASE.11 tool_choice auto/required/named/none — cross-parser suites at + // lib/llm/tests/tool_choice.rs run hermes only; V4 not exercised. + // - CASE.12 FinishReason::Length — current E2E fixtures only cover Stop and + // ToolCalls finish reasons. No truncation-forcing fixture. + // - CASE.14 empty-content / null response at the e2e layer. + // - CASE.15 duplicate-calls (same name twice) — universal gap across all parsers. + // - CASE.16 regression — V4 is hours old (2026-04-24); no customer bugs filed yet. + // ------------------------------------------------------------------- + + /// `CASE.8` — streaming start-token detection (V4 variant). + #[test] + fn test_detect_tool_call_start_v4() { + let config = get_v4_test_config(); + assert!(detect_tool_call_start_dsml("<｜DSML｜tool_calls>", &config)); + assert!(detect_tool_call_start_dsml( + "text <｜DSML｜tool_calls>", + &config + )); + assert!(detect_tool_call_start_dsml("<｜DSML｜tool_c", &config)); + assert!(!detect_tool_call_start_dsml( + "<｜DSML｜function_calls>", + &config + )); + assert!(!detect_tool_call_start_dsml("no tool call here", &config)); + } + #[test] fn test_find_tool_call_end_position() { let config = get_test_config(); @@ -247,6 +365,15 @@ mod tests { assert_eq!(&text[pos..], "more"); } + /// `CASE.8` — streaming end-position lookup (V4 variant). + #[test] + fn test_find_tool_call_end_position_v4() { + let config = get_v4_test_config(); + let text = "<｜DSML｜tool_calls><｜DSML｜invoke name=\"test\">more"; + let pos = find_tool_call_end_position_dsml(text, &config); + assert_eq!(&text[pos..], "more"); + } + #[test] fn test_parse_single_tool_call_string_param() { let input = r#"<｜DSML｜function_calls> @@ -320,6 +447,54 @@ mod tests { assert_eq!(args2["location"], "Hangzhou"); } + /// `CASE.2` multi-calls + `CASE.13` interleaved-text (prefix text before the block). + #[test] + fn test_parse_deepseek_v4_multiple_tool_calls() { + let input = r#"Let's check this. <｜DSML｜tool_calls> +<｜DSML｜invoke name="get_favorite_tourist_spot"> +<｜DSML｜parameter name="city" string="true">Beijing + +<｜DSML｜invoke name="search"> +<｜DSML｜parameter name="query" string="true">search agent benchmark 2024 +<｜DSML｜parameter name="topn" string="false">10 +<｜DSML｜parameter name="source" string="true">web + +"#; + + let config = get_v4_test_config(); + let (calls, normal) = try_tool_call_parse_dsml(input, &config).unwrap(); + assert_eq!(calls.len(), 2); + assert_eq!(normal, Some("Let's check this.".to_string())); + + let (name1, args1) = extract_name_and_args(calls[0].clone()); + assert_eq!(name1, "get_favorite_tourist_spot"); + assert_eq!(args1["city"], "Beijing"); + + let (name2, args2) = extract_name_and_args(calls[1].clone()); + assert_eq!(name2, "search"); + assert_eq!(args2["query"], "search agent benchmark 2024"); + assert_eq!(args2["topn"], 10); + assert_eq!(args2["source"], "web"); + } + + /// `CASE.6` — empty args (no-parameter invoke). + #[test] + fn test_parse_deepseek_v4_no_parameters() { + let input = r#"<｜DSML｜tool_calls> +<｜DSML｜invoke name="get_current_time"> + +"#; + + let config = get_v4_test_config(); + let (calls, normal) = try_tool_call_parse_dsml(input, &config).unwrap(); + assert_eq!(calls.len(), 1); + assert_eq!(normal, Some("".to_string())); + + let (name, args) = extract_name_and_args(calls[0].clone()); + assert_eq!(name, "get_current_time"); + assert_eq!(args, serde_json::json!({})); + } + #[test] fn test_parse_with_normal_text() { let input = r#"Here's the result: <｜DSML｜function_calls> @@ -490,4 +665,116 @@ mod tests { let (_, args) = extract_name_and_args(calls[0].clone()); assert!(args["value"].is_null()); } + + // Corner-case pinning tests. See the V4 coverage manifest above for the + // full mapping from CASE.* → test. Each test's doc-comment names the + // specific CASE it pins. + + /// `CASE.5` — missing end-token recovery. + /// **Pinned as broken** — parser drops the call; see the TODO block above. + /// + /// If a DeepSeek V4 stream is truncated before `` + /// arrives (max_tokens cut-off, EOS mid-generation, connection drop), + /// the block regex requires both fences and matches zero times. The + /// entire DSML-looking payload falls through as raw `normal_text`; no + /// tool calls are recovered. + /// + /// This is the same structural failure mode Kimi K2 had before its + /// parser gained end-token recovery; see + /// `kimi_k2_parser.rs::test_parse_malformed_no_section_end` for the + /// post-fix recovery pattern. + #[test] + fn test_parse_deepseek_v4_missing_end_token() { + // Start fence + complete invoke, but no . + let input = "<｜DSML｜tool_calls>\n\ +<｜DSML｜invoke name=\"get_weather\">\n\ +<｜DSML｜parameter name=\"city\" string=\"true\">NYC\n\ +"; + + let config = get_v4_test_config(); + let (calls, normal_text) = try_tool_call_parse_dsml(input, &config).unwrap(); + + assert!( + calls.is_empty(), + "V4 DSML parser currently drops tool calls when \ + is missing. \ + If recovery is added, flip this assertion." + ); + assert_eq!( + normal_text.as_deref(), + Some(input), + "Unrecovered payload should fall through to normal_text verbatim." + ); + } + + /// `CASE.5` — multiple complete invokes, missing end fence. + /// + /// Even with multiple fully-formed invokes inside the start fence, the + /// absence of the closing fence prevents the block regex from matching. + /// All calls are dropped. If the parser ever gains partial-block + /// recovery, this test will fail and force an intentional update. + #[test] + fn test_parse_deepseek_v4_missing_end_token_multiple_calls() { + let input = "<｜DSML｜tool_calls>\n\ +<｜DSML｜invoke name=\"a\">\n\ +<｜DSML｜parameter name=\"x\" string=\"true\">1\n\ +\n\ +<｜DSML｜invoke name=\"b\">\n\ +<｜DSML｜parameter name=\"y\" string=\"true\">2\n\ +"; + + let config = get_v4_test_config(); + let (calls, _) = try_tool_call_parse_dsml(input, &config).unwrap(); + + assert!( + calls.is_empty(), + "Even two fully-formed invokes are dropped when the outer \ + is missing." + ); + } + + /// `CASE.4` — malformed JSON in a `string="false"` parameter value falls back + /// to a string. `parse_parameters` explicitly swallows the serde error + /// (unwrap_or_else → Value::String). Pin the fallback so removing it + /// (which would cause the whole call to 500 on ragged-edge JSON) is a + /// deliberate change. + #[test] + fn test_parse_deepseek_v4_malformed_json_value_falls_back_to_string() { + let input = "<｜DSML｜tool_calls>\n\ +<｜DSML｜invoke name=\"test\">\n\ +<｜DSML｜parameter name=\"payload\" string=\"false\">{this is not valid json\n\ +\n\ +"; + + let config = get_v4_test_config(); + let (calls, _) = try_tool_call_parse_dsml(input, &config).unwrap(); + assert_eq!(calls.len(), 1); + + let (name, args) = extract_name_and_args(calls[0].clone()); + assert_eq!(name, "test"); + assert_eq!( + args["payload"], "{this is not valid json", + "Malformed JSON should fall back to the raw string, not drop \ + the parameter or the call." + ); + } + + /// `CASE.4` — malformed invoke (missing `` but block fences + /// intact). The invoke regex requires its own close tag, so the call is + /// silently dropped. Pin the behavior. + #[test] + fn test_parse_deepseek_v4_missing_invoke_close_drops_call() { + let input = "<｜DSML｜tool_calls>\n\ +<｜DSML｜invoke name=\"test\">\n\ +<｜DSML｜parameter name=\"x\" string=\"true\">value\n\ +"; + + let config = get_v4_test_config(); + let (calls, _) = try_tool_call_parse_dsml(input, &config).unwrap(); + assert!( + calls.is_empty(), + "Malformed invoke (missing ) is dropped today. \ + If recovery is added, flip this assertion." + ); + } } diff --git a/lib/parsers/src/tool_calling/parsers.rs b/lib/parsers/src/tool_calling/parsers.rs index 5f72bc4c17a4..c3b0c9f34ab1 100644 --- a/lib/parsers/src/tool_calling/parsers.rs +++ b/lib/parsers/src/tool_calling/parsers.rs @@ -43,6 +43,9 @@ pub fn get_tool_parser_map() -> &'static HashMap<&'static str, ToolCallConfig> { map.insert("deepseek_v3", ToolCallConfig::deepseek_v3()); map.insert("deepseek_v3_1", ToolCallConfig::deepseek_v3_1()); map.insert("deepseek_v3_2", ToolCallConfig::deepseek_v3_2()); + map.insert("deepseek_v4", ToolCallConfig::deepseek_v4()); + map.insert("deepseek-v4", ToolCallConfig::deepseek_v4()); + map.insert("deepseekv4", ToolCallConfig::deepseek_v4()); map.insert("qwen3_coder", ToolCallConfig::qwen3_coder()); map.insert("jamba", ToolCallConfig::jamba()); map.insert("minimax_m2", ToolCallConfig::minimax_m2()); @@ -241,6 +244,9 @@ mod tests { "deepseek_v3", "deepseek_v3_1", "deepseek_v3_2", + "deepseek_v4", + "deepseek-v4", + "deepseekv4", "qwen3_coder", "jamba", "nemotron_nano", @@ -1701,6 +1707,56 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it assert_eq!(args["topn"], 10); // Should be number, not string assert_eq!(args["source"], "web"); } + /// `CASE.1` — single-call happy path (V4). + + #[tokio::test] + async fn test_deepseek_v4_single_tool_call() { + let input = r#"<｜DSML｜tool_calls> +<｜DSML｜invoke name="get_datetime"> +<｜DSML｜parameter name="timezone" string="true">Asia/Shanghai + +"#; + + let (tool_calls, normal_text) = + detect_and_parse_tool_call(input, Some("deepseek_v4"), None) + .await + .expect("Failed to parse"); + + assert_eq!(tool_calls.len(), 1); + assert_eq!(tool_calls[0].function.name, "get_datetime"); + assert_eq!(normal_text, Some("".to_string())); + + let args: serde_json::Value = + serde_json::from_str(&tool_calls[0].function.arguments).unwrap(); + assert_eq!(args["timezone"], "Asia/Shanghai"); + } + /// Alias registration: verifies `deepseek-v4` and `deepseekv4` route to the same parser as `deepseek_v4`. Not a CASE.*; covers registry plumbing. + + #[tokio::test] + async fn test_deepseek_v4_compatibility_aliases() { + let input = r#"<｜DSML｜tool_calls> +<｜DSML｜invoke name="search"> +<｜DSML｜parameter name="query" string="true">search agent benchmark 2024 +<｜DSML｜parameter name="topn" string="false">10 +<｜DSML｜parameter name="source" string="true">web + +"#; + + for parser_name in ["deepseek_v4", "deepseek-v4", "deepseekv4"] { + let (tool_calls, _) = detect_and_parse_tool_call(input, Some(parser_name), None) + .await + .expect("Failed to parse"); + + assert_eq!(tool_calls.len(), 1); + assert_eq!(tool_calls[0].function.name, "search"); + + let args: serde_json::Value = + serde_json::from_str(&tool_calls[0].function.arguments).unwrap(); + assert_eq!(args["query"], "search agent benchmark 2024"); + assert_eq!(args["topn"], 10); + assert_eq!(args["source"], "web"); + } + } #[tokio::test] async fn test_hermes_parser_without_new_line() { diff --git a/tests/frontend/test_tool_calling_sglang.py b/tests/frontend/test_tool_calling_sglang.py index a2bdee586c95..daf34cbd5e5a 100644 --- a/tests/frontend/test_tool_calling_sglang.py +++ b/tests/frontend/test_tool_calling_sglang.py @@ -794,24 +794,6 @@ def test_parallel_multi_tool_request_includes_all_expected_tools( names.add(tc["function"]["name"]) assert len(names) >= 2, f"expected at least 2 distinct tools, got {names}" - def test_tool_call_ids_unique_in_single_response(self, client: OpenAI, model: str): - result = stream_chat( - client, - model, - messages=[ - { - "role": "user", - "content": "Get weather for New York, London, and Tokyo.", - } - ], - tools=TOOLS_WEATHER, - tool_choice="required", - parallel_tool_calls=True, - ) - assert_finish_reason(result, {"tool_calls"}) - ids = [tc["id"] for tc in result.tool_calls] - assert len(ids) == len(set(ids)), f"duplicate tool ids: {ids}" - def test_array_argument_schema_valid(self, client: OpenAI, model: str): tools = [ {