[None][fix] DeepSeek DSML tool parsers: the layout newlines around tool calls are not content

tensorrt_llm/serve/tool_parser/deepseekv32_parser.py

@@ -65,6 +65,10 @@ class DeepSeekV32Parser(BaseToolParser):
 
     _eos_token = "<｜end▁of▁sentence｜>"  # nosec B105
 
+    # The encoders write a blank line between a reply's content and its tool-call
+    # block: it belongs to the format, not to the content.
+    _CALLS_SEPARATOR = "\n\n"
+
     # Invoke header up to the function name, which is arbitrary text.
     _INVOKE_HEADER_PREFIX = '<｜DSML｜invoke name="'  # nosec B105
     _INVOKE_BEGIN_TEMPLATE = '<｜DSML｜invoke name="{name}">'  # nosec B105
@@ -80,6 +84,12 @@ def __init__(self):
         )
         self._last_arguments = ""
         self.current_tool_id = -1
+        # Between a block's opening and closing tags, the newlines that separate
+        # the invokes and precede the closing tag are the format's layout (the
+        # checkpoint's encoder writes them), not content.
+        self._in_calls_block = False
+        # Set while an increment re-reads what its last invoke left behind.
+        self._reading_rest = False
 
     def has_tool_call(self, text: str) -> bool:
         """Check if the text contains a deepseek v32 format tool call."""
@@ -129,7 +139,7 @@ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult
         if self._eos_token in text:
             text = text.replace(self._eos_token, "")
         idx = text.find(self.bot_token)
-        normal_text = text[:idx].strip() if idx != -1 else text
+        normal_text = text[:idx].removesuffix(self._CALLS_SEPARATOR) if idx != -1 else text
         if self.bot_token not in text:
             return StreamingParseResult(normal_text=normal_text, calls=[])
 
@@ -163,6 +173,27 @@ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult
             # return the normal text if parsing fails
             return StreamingParseResult(normal_text=text)
 
+    def _separator_prefix_at_end(self, text: str) -> str:
+        """The tail of ``text`` that could still begin the blank line before a calls block."""
+        for size in range(len(self._CALLS_SEPARATOR), 0, -1):
+            if text.endswith(self._CALLS_SEPARATOR[:size]):
+                return text[-size:]
+        return ""
+
+    def finish(self, tools: List[Tool]) -> StreamingParseResult:
+        """Emit, as content, what the stream ended while holding back.
+
+        That is a trailing blank line no tool-call block followed, or a tail that
+        never completed into a delimiter. Inside an unfinished calls block nothing
+        is emitted.
+        """
+        if self._in_calls_block:
+            return StreamingParseResult()
+        held, self._buffer = self._buffer, ""
+        for e_token in [self.eot_token, self.invoke_end_token, self._eos_token]:
+            held = held.replace(e_token, "")
+        return StreamingParseResult(normal_text=held)
+
     def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult:
         """Streaming incremental parsing tool calls for DeepSeekV32 format.
 
@@ -188,9 +219,28 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
             self._ends_with_partial_token(current_text, token) for token in partial_tokens
         )
 
+        if self._in_calls_block and not has_tool_call:
+            closing = current_text.find(self.eot_token)
+            layout = current_text if closing == -1 else current_text[:closing]
+            if not layout.strip():
+                if closing == -1:
+                    # Still layout until the next invoke or the closing tag.
+                    return StreamingParseResult()
+                self._in_calls_block = False
+                current_text = current_text[closing + len(self.eot_token) :]
+                self._buffer = current_text
+                ends_with_partial_token = any(
+                    self._ends_with_partial_token(current_text, token) for token in partial_tokens
+                )
+
         if not has_tool_call and not ends_with_partial_token:
             normal_text = current_text
-            self._buffer = ""
+            # Trailing newlines may begin the blank line before a tool-call block,
+            # which is not content: hold them until the next increment, or
+            # finish(), shows what follows.
+            held = "" if self._in_calls_block else self._separator_prefix_at_end(normal_text)
+            self._buffer = held
+            normal_text = normal_text[: len(normal_text) - len(held)]
             for e_token in [self.eot_token, self.invoke_end_token, self._eos_token]:
                 normal_text = normal_text.replace(e_token, "")
             return StreamingParseResult(normal_text=normal_text)
@@ -202,11 +252,19 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
         if normal_text:
             current_text = current_text[prefix_len:]
             self._buffer = current_text
+        if has_tool_call:
+            if self._in_calls_block and not normal_text.strip():
+                # The separator before the next invoke.
+                normal_text = ""
+            elif not self._in_calls_block and current_text.startswith(self.bot_token):
+                normal_text = normal_text.removesuffix(self._CALLS_SEPARATOR)
+            self._in_calls_block = True
 
         if not hasattr(self, "_tool_indices"):
             self._tool_indices = self._get_tool_indices(tools)
 
         all_calls: list[ToolCallItem] = []
+        invokes_exhausted = False
         try:
             # Loop to handle multiple consecutive invoke blocks
             while True:
@@ -223,6 +281,7 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
                 )
 
                 if not invoke_match:
+                    invokes_exhausted = True
                     break
 
                 func_name = invoke_match.group(1).strip()
@@ -307,7 +366,23 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
                     # Wait for more chunks until we see </｜DSML｜invoke>
                     break
 
-            # No more invoke blocks found
+            # The rest of this increment may close the block and carry text after
+            # it. Read it now: a later increment may never come, and nothing
+            # flushes this parser when the stream ends.
+            if (
+                invokes_exhausted
+                and self._in_calls_block
+                and not self._reading_rest
+                and self.eot_token in self._buffer
+                and not any(token in self._buffer for token in start_tokens)
+            ):
+                self._reading_rest = True
+                try:
+                    rest = self.parse_streaming_increment("", tools)
+                finally:
+                    self._reading_rest = False
+                normal_text += rest.normal_text
+                all_calls.extend(rest.calls)
             return StreamingParseResult(normal_text=normal_text, calls=all_calls)
 
         except Exception as e:

tests/unittest/llmapi/apps/test_tool_parsers.py

@@ -6843,3 +6843,167 @@ def test_parse_base_json_reports_the_name_as_emitted(sample_tools):
 
     assert [(c.name, c.tool_index)
             for c in calls] == [("functions.get_weather", -1)]
+
+
+# ============================================================================
+# DeepSeek DSML layout: the newlines inside a tool-call block are not content
+# ============================================================================
+
+_DSML_FORMATS = [
+    (DeepSeekV32Parser, "function_calls", "invoke", "parameter"),
+    (DeepSeekV4Parser, "tool_calls", "invoke", "parameter"),
+    (DeepSeekV41Parser, " calls", " invoke", " parameter"),
+]
+
+
+def _dsml_block(calls_tag: str, invoke_tag: str, parameter_tag: str) -> str:
+    """Two calls laid out as the checkpoint encoders write them: one per line."""
+    dsml = "｜DSML｜"
+
+    def invoke(name: str, key: str, value: str) -> str:
+        return (f'<{dsml}{invoke_tag} name="{name}">\n'
+                f'<{dsml}{parameter_tag} name="{key}" string="true">{value}'
+                f"</{dsml}{parameter_tag}>\n"
+                f"</{dsml}{invoke_tag}>")
+
+    return (f"<{dsml}{calls_tag}>\n" +
+            invoke("get_weather", "location", "NYC") + "\n" +
+            invoke("search_web", "query", "AI") + "\n" +
+            f"</{dsml}{calls_tag}>")
+
+
+def _stream(parser,
+            text: str,
+            chunk_size: int,
+            tools: list[ChatCompletionToolsParam],
+            *,
+            finish: bool = True):
+    results = [
+        parser.parse_streaming_increment(text[i:i + chunk_size], tools)
+        for i in range(0, len(text), chunk_size)
+    ]
+    if finish:
+        results.append(parser.finish(tools))
+    content = "".join(result.normal_text for result in results)
+    calls = [call for result in results for call in result.calls]
+    return content, calls
+
+
+@pytest.mark.parametrize("parser_cls, calls_tag, invoke_tag, parameter_tag",
+                         _DSML_FORMATS)
+@pytest.mark.parametrize("chunk_size", [1, 3, 7, 10_000])
+def test_dsml_newlines_between_calls_are_not_content(
+        sample_tools: list[ChatCompletionToolsParam],
+        parser_cls: type[BaseToolParser], calls_tag: str, invoke_tag: str,
+        parameter_tag: str, chunk_size: int) -> None:
+    """The newline after each invoke is the block's layout, however it is chunked.
+
+    Streamed as text, it became content after every tool call: over Anthropic
+    Messages, a whitespace-only text block followed each tool_use, and a client
+    that drops such blocks sent back a turn that no longer matched what was
+    served.
+    """
+    text = "Let me check.\n\n" + _dsml_block(calls_tag, invoke_tag,
+                                             parameter_tag)
+
+    content, calls = _stream(parser_cls(), text, chunk_size, sample_tools)
+
+    assert content == "Let me check."
+    assert [call.name for call in calls
+            if call.name] == ["get_weather", "search_web"]
+    arguments = [
+        json.loads(call.parameters) for call in calls
+        if not call.name and call.parameters
+    ]
+    assert arguments == [{"location": "NYC"}, {"query": "AI"}]
+
+
+@pytest.mark.parametrize("parser_cls, calls_tag, invoke_tag, parameter_tag",
+                         _DSML_FORMATS)
+@pytest.mark.parametrize("chunk_size", [5, 10_000])
+def test_dsml_text_after_the_block_is_still_content(
+        sample_tools: list[ChatCompletionToolsParam],
+        parser_cls: type[BaseToolParser], calls_tag: str, invoke_tag: str,
+        parameter_tag: str, chunk_size: int) -> None:
+    """Only the layout inside the block is dropped; text after it is kept.
+
+    In one increment, the closing tag and the text after it follow the last
+    invoke; that text must come out of that increment.
+    """
+    text = _dsml_block(calls_tag, invoke_tag, parameter_tag) + " Done."
+
+    parser = parser_cls()
+    content, calls = _stream(parser,
+                             text,
+                             chunk_size,
+                             sample_tools,
+                             finish=False)
+
+    assert content == " Done."
+    finish_result = parser.finish(sample_tools)
+    assert finish_result.normal_text == ""
+    assert finish_result.calls == []
+    assert [call.name for call in calls
+            if call.name] == ["get_weather", "search_web"]
+
+
+@pytest.mark.parametrize("parser_cls, calls_tag, invoke_tag, parameter_tag",
+                         _DSML_FORMATS)
+@pytest.mark.parametrize("chunk_size", [1, 2, 3, 7, 10_000])
+def test_dsml_blank_line_before_the_block_is_not_content(
+        sample_tools: list[ChatCompletionToolsParam],
+        parser_cls: type[BaseToolParser], calls_tag: str, invoke_tag: str,
+        parameter_tag: str, chunk_size: int) -> None:
+    """The encoders write a blank line between the content and the block.
+
+    The content keeps its own whitespace; only that blank line is dropped, on
+    both paths.
+    """
+    content_in = "  Indented first line.\nLast line.\n"
+    text = content_in + "\n\n" + _dsml_block(calls_tag, invoke_tag,
+                                             parameter_tag)
+
+    content, calls = _stream(parser_cls(), text, chunk_size, sample_tools)
+
+    assert content == content_in
+    assert parser_cls().detect_and_parse(text,
+                                         sample_tools).normal_text == content_in
+    assert [call.name for call in calls
+            if call.name] == ["get_weather", "search_web"]
+
+
+@pytest.mark.parametrize("parser_cls, calls_tag, invoke_tag, parameter_tag",
+                         _DSML_FORMATS)
+@pytest.mark.parametrize("chunk_size", [1, 3, 10_000])
+def test_dsml_trailing_blank_lines_without_a_block_are_content(
+        sample_tools: list[ChatCompletionToolsParam],
+        parser_cls: type[BaseToolParser], calls_tag: str, invoke_tag: str,
+        parameter_tag: str, chunk_size: int) -> None:
+    """Newlines held back in case a block follows come out when none does."""
+    text = "Done.\n\n"
+
+    content, calls = _stream(parser_cls(), text, chunk_size, sample_tools)
+
+    assert content == text
+    assert calls == []
+
+
+@pytest.mark.parametrize("parser_cls, calls_tag, invoke_tag, parameter_tag",
+                         _DSML_FORMATS)
+@pytest.mark.parametrize("chunk_size", [1, 3, 10_000])
+def test_dsml_stream_cut_inside_the_block_keeps_only_the_content(
+        sample_tools: list[ChatCompletionToolsParam],
+        parser_cls: type[BaseToolParser], calls_tag: str, invoke_tag: str,
+        parameter_tag: str, chunk_size: int) -> None:
+    """A stream cut short inside the block ends with its content.
+
+    The non-streaming path reads the same text the same way.
+    """
+    block = _dsml_block(calls_tag, invoke_tag, parameter_tag)
+    text = "Checking.\n\n" + block[:block.index("</｜DSML｜")]
+
+    content, _ = _stream(parser_cls(), text, chunk_size, sample_tools)
+
+    assert content == "Checking."
+    assert parser_cls().detect_and_parse(text,
+                                         sample_tools).normal_text == content