From 0b56da7b4ebae0c2103ffd36c157e3ca84e56688 Mon Sep 17 00:00:00 2001 From: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> Date: Thu, 16 Jul 2026 00:49:13 +0800 Subject: [PATCH] fix(tool_parsers): don't index delta_text into re-serialized JSON in internlm2 parser Internlm2ToolParser.extract_tool_calls_streaming's "first time to get parameters" branch computed the initial arguments delta with: cur_arguments_json[: cur_arguments_json.index(delta_text) + len(delta_text)] This assumes the model's raw delta_text always appears verbatim inside json.dumps(cur_arguments)'s re-serialized string. Once the model's own JSON formatting diverges from Python's default json.dumps spacing (e.g. compact JSON with no space after ':'), .index() raises ValueError, which the caller's broad except Exception swallows into a lost delta -- the arguments for that tool call are permanently dropped, since prev_tool_call_arr is never updated on this path either. Fix by mirroring the sibling "both prev and cur parameters" branch a few lines below, which already uses extract_intermediate_diff(cur, prev) instead of any substring-position assumption: call extract_intermediate_diff(cur_arguments_json, "") for the first-time case (no previous string to diff against), which resolves to the full re-serialized JSON string as intended, without relying on delta_text's formatting. Verified with a standalone repro against a real Internlm2ToolParser instance before touching source: streaming '{"name":"get_weather"' then ',"parameters":{"city":"SF"}}' (compact JSON, no space after ':') raised ValueError: substring not found and returned None for the second delta. Added test_streaming_first_arguments_delta_survives_compact_json_formatting (tests/tool_parsers/test_internlm2_tool_parser.py) covering this exact scenario -- red before the fix, green after. Full tests/tool_parsers/test_internlm2_tool_parser.py: 10 passed, 8 xfailed (pre-existing "InternLM2 streaming not fully implemented" xfail markers in the shared ToolParserTests harness, unrelated to this change -- unchanged before/after). ruff check and ruff format: clean. Duplicate-work check: read the diffs of all 3 currently open PRs touching this file (#44140, #48348, #47968) -- none of their changed line ranges (77/202, 26/79-128, 214) overlap line 150 or the "first time to get parameters" branch; they fix unrelated issues (multi-marker ValueError, whole-call-in-single-delta, unlisted-tool-name). AI-assisted: found via an AI-assisted review pass over vllm/tool_parsers/, independently reproduced, fixed, and verified by me before submitting. Signed-off-by: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> Co-authored-by: Claude Sonnet 5 Signed-off-by: Andrew Chen <48723787+chuenchen309@users.noreply.github.com> --- .../test_internlm2_tool_parser.py | 40 +++++++++++++++++++ vllm/tool_parsers/internlm2_tool_parser.py | 8 ++-- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/tests/tool_parsers/test_internlm2_tool_parser.py b/tests/tool_parsers/test_internlm2_tool_parser.py index 2e5069dbed94..62be0b300493 100644 --- a/tests/tool_parsers/test_internlm2_tool_parser.py +++ b/tests/tool_parsers/test_internlm2_tool_parser.py @@ -10,6 +10,7 @@ ToolParserTests, ) from vllm.tokenizers import TokenizerLike +from vllm.tool_parsers.internlm2_tool_parser import Internlm2ToolParser class TestInternLM2ToolParser(ToolParserTests): @@ -120,3 +121,42 @@ def test_config(self) -> ToolParserTestConfig: ), }, ) + + +def test_streaming_first_arguments_delta_survives_compact_json_formatting(): + """The first streamed arguments delta must not depend on delta_text + appearing verbatim inside json.dumps(cur_arguments): a model emitting + compact JSON (no space after ':') never matches Python's default + json.dumps formatting, so cur_arguments_json.index(delta_text) raises + ValueError, which the caller swallows into a lost delta. + """ + parser = Internlm2ToolParser(tokenizer=None, tools=[]) + + first = parser.extract_tool_calls_streaming( + previous_text="", + current_text='<|action_start|><|plugin|>{"name":"get_weather"', + delta_text='<|action_start|><|plugin|>{"name":"get_weather"', + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=None, + ) + assert first is not None + assert first.tool_calls[0].function.name == "get_weather" + + second = parser.extract_tool_calls_streaming( + previous_text='<|action_start|><|plugin|>{"name":"get_weather"', + current_text=( + '<|action_start|><|plugin|>{"name":"get_weather",' + '"parameters":{"city":"SF"}}' + ), + delta_text=',"parameters":{"city":"SF"}}', + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=None, + ) + + assert second is not None + assert second.tool_calls[0].function.arguments is not None + assert parser.streamed_args_for_tool[0] != "" diff --git a/vllm/tool_parsers/internlm2_tool_parser.py b/vllm/tool_parsers/internlm2_tool_parser.py index f4aaeef71a0f..60f937d7625b 100644 --- a/vllm/tool_parsers/internlm2_tool_parser.py +++ b/vllm/tool_parsers/internlm2_tool_parser.py @@ -146,9 +146,11 @@ def extract_tool_calls_streaming( elif cur_arguments and not prev_arguments: cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False) - arguments_delta = cur_arguments_json[ - : cur_arguments_json.index(delta_text) + len(delta_text) - ] + # delta_text may not appear verbatim in cur_arguments_json + # (e.g. compact model output vs. json.dumps spacing), so + # diff against an empty previous string instead of + # locating delta_text as a substring. + arguments_delta = extract_intermediate_diff(cur_arguments_json, "") delta = DeltaMessage( tool_calls=[ DeltaToolCall(