diff --git a/tests/tool_parsers/test_internlm2_tool_parser.py b/tests/tool_parsers/test_internlm2_tool_parser.py index 2e5069dbed94..62be0b300493 100644 --- a/tests/tool_parsers/test_internlm2_tool_parser.py +++ b/tests/tool_parsers/test_internlm2_tool_parser.py @@ -10,6 +10,7 @@ ToolParserTests, ) from vllm.tokenizers import TokenizerLike +from vllm.tool_parsers.internlm2_tool_parser import Internlm2ToolParser class TestInternLM2ToolParser(ToolParserTests): @@ -120,3 +121,42 @@ def test_config(self) -> ToolParserTestConfig: ), }, ) + + +def test_streaming_first_arguments_delta_survives_compact_json_formatting(): + """The first streamed arguments delta must not depend on delta_text + appearing verbatim inside json.dumps(cur_arguments): a model emitting + compact JSON (no space after ':') never matches Python's default + json.dumps formatting, so cur_arguments_json.index(delta_text) raises + ValueError, which the caller swallows into a lost delta. + """ + parser = Internlm2ToolParser(tokenizer=None, tools=[]) + + first = parser.extract_tool_calls_streaming( + previous_text="", + current_text='<|action_start|><|plugin|>{"name":"get_weather"', + delta_text='<|action_start|><|plugin|>{"name":"get_weather"', + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=None, + ) + assert first is not None + assert first.tool_calls[0].function.name == "get_weather" + + second = parser.extract_tool_calls_streaming( + previous_text='<|action_start|><|plugin|>{"name":"get_weather"', + current_text=( + '<|action_start|><|plugin|>{"name":"get_weather",' + '"parameters":{"city":"SF"}}' + ), + delta_text=',"parameters":{"city":"SF"}}', + previous_token_ids=[], + current_token_ids=[], + delta_token_ids=[], + request=None, + ) + + assert second is not None + assert second.tool_calls[0].function.arguments is not None + assert parser.streamed_args_for_tool[0] != "" diff --git a/vllm/tool_parsers/internlm2_tool_parser.py b/vllm/tool_parsers/internlm2_tool_parser.py index f4aaeef71a0f..60f937d7625b 100644 --- a/vllm/tool_parsers/internlm2_tool_parser.py +++ b/vllm/tool_parsers/internlm2_tool_parser.py @@ -146,9 +146,11 @@ def extract_tool_calls_streaming( elif cur_arguments and not prev_arguments: cur_arguments_json = json.dumps(cur_arguments, ensure_ascii=False) - arguments_delta = cur_arguments_json[ - : cur_arguments_json.index(delta_text) + len(delta_text) - ] + # delta_text may not appear verbatim in cur_arguments_json + # (e.g. compact model output vs. json.dumps spacing), so + # diff against an empty previous string instead of + # locating delta_text as a substring. + arguments_delta = extract_intermediate_diff(cur_arguments_json, "") delta = DeltaMessage( tool_calls=[ DeltaToolCall(