From 00816550e740a13736f7ef9921097d51a3f01ede Mon Sep 17 00:00:00 2001 From: "Eugenio \"Jay\" Zuccarelli" <11176606+jayzuccarelli@users.noreply.github.com> Date: Fri, 25 Sep 2026 18:34:23 +0000 Subject: [PATCH 1/2] [Bugfix][Parser] Stream DeepSeek V4/V3.2 string tool arguments incrementally DeepSeek V4/V3.2 set arg_structural_chars=frozenset(">"), so ParserEngine._compute_arg_delta skipped the converter for every parameter-body delta and a long string argument was buffered until . Only apply the structural gate when the tool has no streamable string keys. The gate was also hiding a trailing, still-unclosed next parameter tag from the DSML partial regex; hold it back so an implicitly closed string value never streams the tag. Fixes #52846 Co-authored-by: Claude Opus 5.5 Signed-off-by: Eugenio "Jay" Zuccarelli <11176606+jayzuccarelli@users.noreply.github.com> --- .../test_deepseekv32_tool_parser.py | 39 +++++++++++ .../test_deepseekv4_tool_parser.py | 65 +++++++++++++++++++ vllm/parser/deepseek_v4.py | 4 +- vllm/parser/engine/parser_engine.py | 12 +++- 4 files changed, 117 insertions(+), 3 deletions(-) diff --git a/tests/tool_parsers/test_deepseekv32_tool_parser.py b/tests/tool_parsers/test_deepseekv32_tool_parser.py index 40ad6033d3bf..3783ca8a78bb 100644 --- a/tests/tool_parsers/test_deepseekv32_tool_parser.py +++ b/tests/tool_parsers/test_deepseekv32_tool_parser.py @@ -969,6 +969,45 @@ def test_emits_arguments_before_invoke_completes(self, parser): assert combined # some partial args emitted assert combined.startswith('{"k"') + def test_long_string_argument_streams_before_close(self): + """A string body has no ">" and must not be buffered until + arrives (GitHub #52846).""" + tool = ChatCompletionToolsParam( + function=FunctionDefinition( + name="emit", + parameters={ + "type": "object", + "properties": {"text": {"type": "string"}}, + }, + ), + ) + parser = make_parser(tools=[tool]) + body = "A" * 4096 + chunks = [ + f'{FC_START}\n{INV_START}emit">\n{PARAM_START}text" string="true">', + *(body[i : i + 32] for i in range(0, len(body), 32)), + f"{PARAM_END}\n{INV_END}\n{FC_END}", + ] + per_chunk_args = [] + prev = "" + for chunk in chunks: + result = parser.extract_tool_calls_streaming( + previous_text=prev, + current_text=prev + chunk, + delta_text=chunk, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[1], + request=make_request(), + ) + prev += chunk + per_chunk_args.append( + self._reconstruct_args([result]) if result is not None else "" + ) + + assert sum(1 for args in per_chunk_args[1:-1] if args) > 1 + assert json.loads("".join(per_chunk_args)) == {"text": body} + def test_no_marker_leak_chunked(self, parser): """Chunked streaming must NOT leak DSML start-marker fragments as content (GitHub #40801).""" diff --git a/tests/tool_parsers/test_deepseekv4_tool_parser.py b/tests/tool_parsers/test_deepseekv4_tool_parser.py index e79465ed9b32..2e25f4193aa2 100644 --- a/tests/tool_parsers/test_deepseekv4_tool_parser.py +++ b/tests/tool_parsers/test_deepseekv4_tool_parser.py @@ -254,6 +254,71 @@ def test_streaming_emits_incremental_argument_chunks(): } +def _stream_param_body(properties: dict, param_open: str, body: str) -> list[str]: + """Stream one parameter body in 32-char chunks, one argument list per chunk.""" + tool = ChatCompletionToolsParam( + function=FunctionDefinition( + name="emit", + parameters={"type": "object", "properties": properties}, + ), + ) + parser = make_parser(tools=[tool]) + chunks = [ + f'{TC_START}\n{INV_START}emit">\n{param_open}', + *(body[i : i + 32] for i in range(0, len(body), 32)), + f"{PARAM_END}\n{INV_END}\n{TC_END}", + ] + per_chunk_args = [] + previous_text = "" + for delta_text in chunks: + current_text = previous_text + delta_text + delta = parser.extract_tool_calls_streaming( + previous_text=previous_text, + current_text=current_text, + delta_text=delta_text, + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[1], + request=make_request(), + ) + previous_text = current_text + per_chunk_args.append( + "".join( + tool_call.function.arguments + for tool_call in (delta.tool_calls if delta else []) + if tool_call.function and tool_call.function.arguments + ) + ) + return per_chunk_args + + +def test_streaming_long_string_argument_is_incremental(): + # Regression for #52846: a string body has no ">" so it used to be + # buffered until arrived. + body = "A" * 4096 + per_chunk_args = _stream_param_body( + {"text": {"type": "string"}}, + f'{PARAM_START}text" string="true">', + body, + ) + + body_chunks = per_chunk_args[1:-1] + assert sum(1 for args in body_chunks if args) > 1 + assert json.loads("".join(per_chunk_args)) == {"text": body} + + +def test_streaming_non_string_argument_is_withheld_until_close(): + body = "1" * 128 + per_chunk_args = _stream_param_body( + {"count": {"type": "integer"}}, + f'{PARAM_START}count" string="false">', + body, + ) + + assert not any(per_chunk_args[1:-1]) + assert json.loads("".join(per_chunk_args)) == {"count": int(body)} + + def _with_strict( tools: list[ChatCompletionToolsParam], ) -> list[ChatCompletionToolsParam]: diff --git a/vllm/parser/deepseek_v4.py b/vllm/parser/deepseek_v4.py index 49ef63736531..03d4a6471903 100644 --- a/vllm/parser/deepseek_v4.py +++ b/vllm/parser/deepseek_v4.py @@ -64,12 +64,14 @@ def _param_patterns( The ``string`` attribute is optional: the model sometimes omits it, and dropping such a parameter hands the client a tool call with no arguments. + The trailing-partial regex holds back an unfinished next parameter tag + so a streamed string value never includes it. """ start, close = re.escape(param_start), re.escape(param_close) head = rf'{start}\s+name="([^"]+)"(?:\s+string="([^"]*)")?\s*>' return ( re.compile(rf"{head}(.*?)(?:{close}|(?={start}\s+name=))", re.DOTALL), - re.compile(rf"{head}(.*)$", re.DOTALL), + re.compile(rf"{head}(.*?)(?:{start}[^>]*)?$", re.DOTALL), ) diff --git a/vllm/parser/engine/parser_engine.py b/vllm/parser/engine/parser_engine.py index 527c310c1485..deb5774dcde2 100644 --- a/vllm/parser/engine/parser_engine.py +++ b/vllm/parser/engine/parser_engine.py @@ -1014,11 +1014,19 @@ def _compute_arg_delta(self, idx: int, raw_delta: str) -> str | None: if not self._stream_arg_deltas: return None + slot = self._tool_slots[idx] + # Structural chars gate the converter call, but a string value can + # grow without them. Only gate when ``string_keys`` is ``set()`` (the + # schema has no string fields); ``None`` means no schema, so every + # string value is streamable. structural = self._arg_structural_chars - if structural is not None and structural.isdisjoint(raw_delta): + if ( + structural is not None + and slot.string_keys == set() + and structural.isdisjoint(raw_delta) + ): return None - slot = self._tool_slots[idx] try: current_json = converter(slot.args, True) except (json.JSONDecodeError, ValueError, TypeError): From 3062cb2fa043a5f96f11cb2d0e6659f053cf575b Mon Sep 17 00:00:00 2001 From: "Eugenio \"Jay\" Zuccarelli" <11176606+jayzuccarelli@users.noreply.github.com> Date: Thu, 15 Oct 2026 09:36:03 +0000 Subject: [PATCH 2/2] [Bugfix][Parser] Stream DSML string arguments via arg_value_extender Upstream added a per-parser arg_value_extender hook (#61253 for GLM, #61494 for Gemma4), so drop the engine-level gate change and port the fix onto that hook instead. _dsml_arg_value_extender returns an extender while the last parameter is an open string="true" value, so each delta streams as escaped JSON text instead of waiting for a ">" to rerun the converter. It keeps a tail to stop on a parameter tag split across deltas. Wired into the V3.2, V4 and V4.1 (spaced DSML) configs. Adds a V4.1 long-string streaming test, extender unit tests, and a string-with-markup streaming test. Co-authored-by: Claude Opus 5.5 Signed-off-by: Eugenio "Jay" Zuccarelli <11176606+jayzuccarelli@users.noreply.github.com> --- tests/parser/engine/test_deepseek_v4.py | 34 +++++++++++++++ tests/parser/engine/test_deepseek_v41.py | 37 ++++++++++++++++ .../test_deepseekv4_tool_parser.py | 13 ++++++ vllm/parser/deepseek_v32.py | 2 + vllm/parser/deepseek_v4.py | 42 +++++++++++++++++++ vllm/parser/deepseek_v41.py | 8 ++++ 6 files changed, 136 insertions(+) diff --git a/tests/parser/engine/test_deepseek_v4.py b/tests/parser/engine/test_deepseek_v4.py index f5b66d9dc1cb..a8227c777428 100644 --- a/tests/parser/engine/test_deepseek_v4.py +++ b/tests/parser/engine/test_deepseek_v4.py @@ -33,6 +33,7 @@ DSML_TOOL_START_VARIANTS, DeepSeekV4Parser, _dsml_arg_converter, + _dsml_arg_value_extender, _unwrap_wrapper_args, deepseek_v4_config, ) @@ -1302,6 +1303,39 @@ def test_eos_not_leaked_when_reasoning_never_ends(self, chunk_size): assert output.tool_calls == [] +# ── Open string values stream without re-running the converter (#52846) ── + + +class TestArgValueExtender: + def test_stops_at_split_param_close(self): + extend = _dsml_arg_value_extender( + "<" + _PARAM_OPEN.format(name="k", is_str="true") + "x" + ) + assert [extend(d) for d in ['a"\n', ""]] == [ + 'a\\"\\n', + "" so it used to be + # buffered until the closing parameter tag arrived. + body = "A" * 4096 + text = ( + '<｜DSML｜ calls>\n<｜DSML｜ invoke name="emit">\n' + f'<｜DSML｜ parameter name="text" string="true">{body}' + "\n\n" + ) + tools = [ + { + "type": "function", + "function": { + "name": "emit", + "parameters": { + "type": "object", + "properties": {"text": {"type": "string"}}, + }, + }, + } + ] + tokenizer, tokens = tokenizer_for(text, False) + parser = parser_for(tokenizer, {"thinking": False}) + results = replay_streaming( + parser, tokens, chunk_size=32, finished_on_last=True, tools=tools + ) + arg_chunks = [ + tool_call.function.arguments + for result in results + if result + for tool_call in result.tool_calls or [] + if tool_call.function and tool_call.function.arguments + ] + assert len(arg_chunks) > len(body) // 64 + assert json.loads("".join(arg_chunks)) == {"text": body} diff --git a/tests/tool_parsers/test_deepseekv4_tool_parser.py b/tests/tool_parsers/test_deepseekv4_tool_parser.py index 2e25f4193aa2..f730e29b9ad5 100644 --- a/tests/tool_parsers/test_deepseekv4_tool_parser.py +++ b/tests/tool_parsers/test_deepseekv4_tool_parser.py @@ -319,6 +319,19 @@ def test_streaming_non_string_argument_is_withheld_until_close(): assert json.loads("".join(per_chunk_args)) == {"count": int(body)} +def test_streaming_string_argument_with_markup_is_incremental(): + body = '
x > y
\n' * 64 + per_chunk_args = _stream_param_body( + {"html": {"type": "string"}}, + f'{PARAM_START}html" string="true">', + body, + ) + + body_chunks = per_chunk_args[1:-1] + assert sum(1 for args in body_chunks if args) > 1 + assert json.loads("".join(per_chunk_args)) == {"html": body} + + def _with_strict( tools: list[ChatCompletionToolsParam], ) -> list[ChatCompletionToolsParam]: diff --git a/vllm/parser/deepseek_v32.py b/vllm/parser/deepseek_v32.py index de6634cf7368..ae5f33a09961 100644 --- a/vllm/parser/deepseek_v32.py +++ b/vllm/parser/deepseek_v32.py @@ -28,6 +28,7 @@ DSML_PARAM_CLOSE, DSML_PARAM_START, _dsml_arg_converter, + _dsml_arg_value_extender, _unwrap_wrapper_args, ) from vllm.parser.engine.events import EventType @@ -115,6 +116,7 @@ def deepseek_v32_config() -> ParserEngineConfig: }, arg_converter=_dsml_arg_converter, arg_structural_chars=frozenset(">"), + arg_value_extender=_dsml_arg_value_extender, strip_content_whitespace_with_tools=False, tool_args_json=False, ) diff --git a/vllm/parser/deepseek_v4.py b/vllm/parser/deepseek_v4.py index 03d4a6471903..2caed0dff55e 100644 --- a/vllm/parser/deepseek_v4.py +++ b/vllm/parser/deepseek_v4.py @@ -28,6 +28,7 @@ from vllm.parser.engine.events import EventType from vllm.parser.engine.parser_engine import ParserEngine from vllm.parser.engine.parser_engine_config import ( + ArgValueExtender, ParserEngineConfig, ParserState, Transition, @@ -114,6 +115,46 @@ def _dsml_arg_converter( return json.dumps(params, ensure_ascii=False) +def _dsml_arg_value_extender( + raw_args: str, + *, + param_re: re.Pattern = _PARAM_RE, + partial_param_re: re.Pattern = _PARTIAL_PARAM_RE, + param_start: str = DSML_PARAM_START, + param_close: str = DSML_PARAM_CLOSE, +) -> ArgValueExtender | None: + # ``arg_structural_chars`` skips every delta without ``>``, so a long + # string value would wait for its closing tag. Extend the open trailing + # ``string="true"`` value instead. The lexer can flush a held partial + # tag as text, so keep a tail to catch a tag split across deltas. + names = set() + last_end = 0 + for m in param_re.finditer(raw_args): + names.add(m.group(1)) + last_end = m.end() + pm = partial_param_re.search(raw_args, last_end) + if ( + pm is None + or pm.group(2) != "true" + or pm.group(1) in names + or pm.end(3) != len(raw_args) + ): + return None + + keep = max(len(param_start), len(param_close)) - 1 + tail = raw_args[-keep:] + + def extend(delta: str) -> str | None: + nonlocal tail + text = tail + delta + if param_start in text or param_close in text: + return None + tail = text[-keep:] + return json.dumps(delta, ensure_ascii=False)[1:-1] + + return extend + + def _unwrap_wrapper_args( args_json: str, tools: list[Tool] | None, @@ -240,6 +281,7 @@ def deepseek_v4_config(thinking: bool = False) -> ParserEngineConfig: }, arg_converter=_dsml_arg_converter, arg_structural_chars=frozenset(">"), + arg_value_extender=_dsml_arg_value_extender, strip_content_whitespace_with_tools=False, tool_args_json=False, ) diff --git a/vllm/parser/deepseek_v41.py b/vllm/parser/deepseek_v41.py index 84216005102a..c3293833fbf1 100644 --- a/vllm/parser/deepseek_v41.py +++ b/vllm/parser/deepseek_v41.py @@ -8,6 +8,7 @@ from vllm.parser.deepseek_v4 import ( DeepSeekV4Parser, _dsml_arg_converter, + _dsml_arg_value_extender, _param_patterns, deepseek_v4_config, ) @@ -47,6 +48,13 @@ def deepseek_v41_config(thinking: bool = False) -> ParserEngineConfig: param_re=_PARAM_RE, partial_param_re=_PARTIAL_PARAM_RE, ), + arg_value_extender=functools.partial( + _dsml_arg_value_extender, + param_re=_PARAM_RE, + partial_param_re=_PARTIAL_PARAM_RE, + param_start=DSML_PARAM_START, + param_close=DSML_PARAM_CLOSE, + ), )