Skip to content

[#17580][fix] Stream the text that precedes a tool call in DeepSeek parsers #17903

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tongyuantongyu merged 3 commits into from
Aug 19, 2026
Merged

[#17580][fix] Stream the text that precedes a tool call in DeepSeek parsers #17903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bb812e8
[#17580][fix] Stream the text that precedes a tool call in DeepSeek p…
Yigtwxx Aug 18, 2026
fc96d53
[#17580][test] Pin the streamed prefix as delta-independent
Yigtwxx Aug 18, 2026
fd7c4e8
[#17580][test] Assert the streamed tool call survives the prefix
Yigtwxx Aug 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 15 additions & 6 deletions tensorrt_llm/serve/tool_parser/deepseekv31_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,12 +95,13 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
current_text = self._buffer

# Check if we have a tool call (either the start token or individual tool call)
has_tool_call = self.bot_token in current_text or "<｜tool▁call▁begin｜>" in current_text
start_tokens = [self.bot_token, "<｜tool▁call▁begin｜>"]
start_indices = [idx for idx in map(current_text.find, start_tokens) if idx != -1]
has_tool_call = bool(start_indices)

if not has_tool_call:
if any(
self._ends_with_partial_token(current_text, b_token)
for b_token in [self.bot_token, "<｜tool▁call▁begin｜>"]
self._ends_with_partial_token(current_text, b_token) for b_token in start_tokens
):
return StreamingParseResult()
normal_text = current_text
Expand All @@ -109,6 +110,14 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
normal_text = normal_text.replace(e_token, "")
return StreamingParseResult(normal_text=normal_text)

# Text before the earliest start token is content, so stream it and keep the
# buffer from that token onwards.
prefix_len = min(start_indices, default=0)
normal_text = current_text[:prefix_len]
if normal_text:
current_text = current_text[prefix_len:]
self._buffer = current_text

if not hasattr(self, "_tool_indices"):
self._tool_indices = self._get_tool_indices(tools)

Expand Down Expand Up @@ -183,17 +192,17 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
else:
self._buffer = ""

result = StreamingParseResult(normal_text="", calls=calls)
result = StreamingParseResult(normal_text=normal_text, calls=calls)
self.current_tool_id += 1
self._last_arguments = ""
self.current_tool_name_sent = False
return result

return StreamingParseResult(normal_text="", calls=calls)
return StreamingParseResult(normal_text=normal_text, calls=calls)

except Exception as e:
logger.error(f"Error in parse_streaming_increment: {e}")
return StreamingParseResult(normal_text=current_text)
return StreamingParseResult(normal_text=normal_text + current_text)

def structure_info(self) -> _GetInfoFunc:
return lambda name: StructureInfo(
Expand Down
16 changes: 13 additions & 3 deletions tensorrt_llm/serve/tool_parser/deepseekv32_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,9 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
self._buffer += new_text
current_text = self._buffer

has_tool_call = self.bot_token in current_text or self._INVOKE_HEADER_PREFIX in current_text
start_tokens = [self.bot_token, self._INVOKE_HEADER_PREFIX]
start_indices = [idx for idx in map(current_text.find, start_tokens) if idx != -1]
has_tool_call = bool(start_indices)

# Hold the buffer back only while its tail could still complete into one of
# the DSML delimiters.
Expand All @@ -192,6 +194,14 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
normal_text = normal_text.replace(e_token, "")
return StreamingParseResult(normal_text=normal_text)

# Text before the earliest start token is content, so stream it and keep the
# buffer from that token onwards.
prefix_len = min(start_indices, default=0)
normal_text = current_text[:prefix_len]
if normal_text:
current_text = current_text[prefix_len:]
self._buffer = current_text

if not hasattr(self, "_tool_indices"):
self._tool_indices = self._get_tool_indices(tools)

Expand Down Expand Up @@ -292,11 +302,11 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
break

# No more invoke blocks found
return StreamingParseResult(normal_text="", calls=all_calls)
return StreamingParseResult(normal_text=normal_text, calls=all_calls)

except Exception as e:
logger.error(f"Error in parse_streaming_increment: {e}")
return StreamingParseResult(normal_text=current_text)
return StreamingParseResult(normal_text=normal_text + current_text)

def structure_info(self) -> _GetInfoFunc:
return lambda name: StructureInfo(
Expand Down
21 changes: 15 additions & 6 deletions tensorrt_llm/serve/tool_parser/deepseekv3_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,12 +98,13 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
current_text = self._buffer

# Check if we have a tool call (either the start token or individual tool call)
has_tool_call = self.bot_token in current_text or "<｜tool▁call▁begin｜>" in current_text
start_tokens = [self.bot_token, "<｜tool▁call▁begin｜>"]
start_indices = [idx for idx in map(current_text.find, start_tokens) if idx != -1]
has_tool_call = bool(start_indices)

if not has_tool_call:
if any(
self._ends_with_partial_token(current_text, b_token)
for b_token in [self.bot_token, "<｜tool▁call▁begin｜>"]
self._ends_with_partial_token(current_text, b_token) for b_token in start_tokens
):
return StreamingParseResult()
normal_text = current_text
Expand All @@ -112,6 +113,14 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
normal_text = normal_text.replace(e_token, "")
return StreamingParseResult(normal_text=normal_text)

# Text before the earliest start token is content, so stream it and keep the
# buffer from that token onwards.
prefix_len = min(start_indices, default=0)
normal_text = current_text[:prefix_len]
if normal_text:
current_text = current_text[prefix_len:]
self._buffer = current_text

Comment thread
coderabbitai[bot] marked this conversation as resolved.
if not hasattr(self, "_tool_indices"):
self._tool_indices = self._get_tool_indices(tools)

Expand Down Expand Up @@ -187,17 +196,17 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
else:
self._buffer = ""

result = StreamingParseResult(normal_text="", calls=calls)
result = StreamingParseResult(normal_text=normal_text, calls=calls)
self.current_tool_id += 1
self._last_arguments = ""
self.current_tool_name_sent = False
return result

return StreamingParseResult(normal_text="", calls=calls)
return StreamingParseResult(normal_text=normal_text, calls=calls)

except Exception as e:
logger.error(f"Error in parse_streaming_increment: {e}")
return StreamingParseResult(normal_text=current_text)
return StreamingParseResult(normal_text=normal_text + current_text)
Comment thread
coderabbitai[bot] marked this conversation as resolved.

def structure_info(self) -> _GetInfoFunc:
return lambda name: StructureInfo(
Expand Down
96 changes: 96 additions & 0 deletions tests/unittest/llmapi/apps/test_tool_parsers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1814,6 +1814,102 @@ def test_deepseek_streaming_preserves_withheld_text(
sample_tools).normal_text == expected


@pytest.mark.parametrize(
"parser_cls, tool_call_text",
[
(
DeepSeekV3Parser,
("<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>"
'get_weather\n```json\n{"location": "Tokyo"}\n```'
"<｜tool▁call▁end｜><｜tool▁calls▁end｜>"),
),
(
DeepSeekV31Parser,
("<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>"
'{"location": "Tokyo"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>'),
),
(
DeepSeekV32Parser,
('<｜DSML｜function_calls><｜DSML｜invoke name="get_weather">'
'<｜DSML｜parameter name="location" string="true">Tokyo'
"</｜DSML｜parameter></｜DSML｜invoke></｜DSML｜function_calls>"),
),
(
DeepSeekV4Parser,
('<｜DSML｜tool_calls><｜DSML｜invoke name="get_weather">'
'<｜DSML｜parameter name="location" string="true">Tokyo'
"</｜DSML｜parameter></｜DSML｜invoke></｜DSML｜tool_calls>"),
),
],
)
def test_deepseek_streaming_emits_text_before_tool_call(
sample_tools: list[ChatCompletionToolsParam],
parser_cls: type[BaseToolParser], tool_call_text: str) -> None:
"""Text that precedes a tool call in the same delta is content."""
text = "Normal text" + tool_call_text

result = parser_cls().parse_streaming_increment(text, sample_tools)

assert result.normal_text == "Normal text"
assert result.normal_text == parser_cls().detect_and_parse(
text, sample_tools).normal_text
assert "get_weather" in [call.name for call in result.calls if call.name]


@pytest.mark.parametrize(
"parser_cls, tool_call_text",
[
(
DeepSeekV3Parser,
("<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>"
'get_weather\n```json\n{"location": "Tokyo"}\n```'
"<｜tool▁call▁end｜><｜tool▁calls▁end｜>"),
),
(
DeepSeekV31Parser,
("<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>get_weather<｜tool▁sep｜>"
'{"location": "Tokyo"}<｜tool▁call▁end｜><｜tool▁calls▁end｜>'),
),
(
DeepSeekV32Parser,
('<｜DSML｜function_calls><｜DSML｜invoke name="get_weather">'
'{"location": "Tokyo"}</｜DSML｜invoke></｜DSML｜function_calls>'),
),
(
DeepSeekV4Parser,
('<｜DSML｜tool_calls><｜DSML｜invoke name="get_weather">'
'{"location": "Tokyo"}</｜DSML｜invoke></｜DSML｜tool_calls>'),
),
],
)
def test_deepseek_streaming_prefix_is_delta_independent(
sample_tools: list[ChatCompletionToolsParam],
parser_cls: type[BaseToolParser], tool_call_text: str) -> None:
"""The prefix is streamed verbatim however the deltas are cut."""
prefix = " Normal text "
text = prefix + tool_call_text
splits = [
[text],
[prefix, tool_call_text],
[text[:8], text[8:]],
]

for deltas in splits:
parser = parser_cls()
results = [
parser.parse_streaming_increment(delta, sample_tools)
for delta in deltas
]
streamed = "".join(result.normal_text for result in results)
names = [
call.name for result in results for call in result.calls
if call.name
]

assert streamed == prefix, f"{deltas!r} streamed {streamed!r}"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
assert names == ["get_weather"], f"{deltas!r} called {names!r}"


# ============================================================================
# Glm4ToolParser Tests
# ============================================================================
Expand Down
Loading