diff --git a/tests/tool_parsers/test_llama4_pythonic_tool_parser.py b/tests/tool_parsers/test_llama4_pythonic_tool_parser.py index 351e4115a6fc..16560a068b56 100644 --- a/tests/tool_parsers/test_llama4_pythonic_tool_parser.py +++ b/tests/tool_parsers/test_llama4_pythonic_tool_parser.py @@ -267,3 +267,59 @@ def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLik assert content == fake_problematic_input assert len(tool_calls) == 0 mock_regex.match.assert_called_once() + + +TRAILING_TEXT_CASES = [ + pytest.param( + f"{SIMPLE_FUNCTION_OUTPUT} Let me check that for you.", + "Let me check that for you.", + id="same_line", + ), + pytest.param( + f"{SIMPLE_FUNCTION_OUTPUT}\n\nI'll wait for the result.", + "I'll wait for the result.", + id="after_blank_line", + ), +] + + +@pytest.mark.parametrize("streaming", [True, False]) +@pytest.mark.parametrize("model_output, expected_content", TRAILING_TEXT_CASES) +def test_tool_call_followed_by_text( + streaming: bool, + model_output: str, + expected_content: str, + default_tokenizer: TokenizerLike, +): + """Both modes return the call and the text after it.""" + tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( + default_tokenizer + ) + + content, tool_calls = run_tool_extraction( + tool_parser, model_output, streaming=streaming + ) + + assert content == expected_content + assert len(tool_calls) == 1 + assert tool_calls[0].function == SIMPLE_FUNCTION_CALL + + +@pytest.mark.parametrize("streaming", [True, False]) +def test_leading_underscore_identifiers( + streaming: bool, default_tokenizer: TokenizerLike +): + """Leading-underscore names are valid Python and JSON Schema names.""" + tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( + default_tokenizer + ) + + content, tool_calls = run_tool_extraction( + tool_parser, "[_lookup(_id='abc', limit=5)]", streaming=streaming + ) + + assert content is None + assert len(tool_calls) == 1 + assert tool_calls[0].function == FunctionCall( + name="_lookup", arguments='{"_id": "abc", "limit": 5}' + ) diff --git a/tests/tool_parsers/test_olmo3_tool_parser.py b/tests/tool_parsers/test_olmo3_tool_parser.py index 02b5c621f61a..8c43bbe8c615 100644 --- a/tests/tool_parsers/test_olmo3_tool_parser.py +++ b/tests/tool_parsers/test_olmo3_tool_parser.py @@ -249,3 +249,25 @@ def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLik assert content == fake_problematic_input assert len(tool_calls) == 0 mock_regex.match.assert_called_once() + + +@pytest.mark.parametrize("streaming", [True, False]) +def test_leading_underscore_identifiers( + streaming: bool, default_tokenizer: TokenizerLike +): + """Leading-underscore names are valid Python and JSON Schema names.""" + tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( + default_tokenizer + ) + + content, tool_calls = run_tool_extraction( + tool_parser, + "_lookup(_id='abc', limit=5)", + streaming=streaming, + ) + + assert content is None + assert len(tool_calls) == 1 + assert tool_calls[0].function == FunctionCall( + name="_lookup", arguments='{"_id": "abc", "limit": 5}' + ) diff --git a/tests/tool_parsers/test_pythonic_tool_parser.py b/tests/tool_parsers/test_pythonic_tool_parser.py index fec713d5a78d..d3b818836190 100644 --- a/tests/tool_parsers/test_pythonic_tool_parser.py +++ b/tests/tool_parsers/test_pythonic_tool_parser.py @@ -229,3 +229,59 @@ def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLik assert content == fake_problematic_input assert len(tool_calls) == 0 mock_regex.match.assert_called_once() + + +TRAILING_TEXT_CASES = [ + pytest.param( + f"[{SIMPLE_FUNCTION_OUTPUT}] Let me check that for you.", + "Let me check that for you.", + id="same_line", + ), + pytest.param( + f"[{SIMPLE_FUNCTION_OUTPUT}]\n\nI'll wait for the result.", + "I'll wait for the result.", + id="after_blank_line", + ), +] + + +@pytest.mark.parametrize("streaming", [True, False]) +@pytest.mark.parametrize("model_output, expected_content", TRAILING_TEXT_CASES) +def test_tool_call_followed_by_text( + streaming: bool, + model_output: str, + expected_content: str, + default_tokenizer: TokenizerLike, +): + """Both modes return the call and the text after it.""" + tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( + default_tokenizer + ) + + content, tool_calls = run_tool_extraction( + tool_parser, model_output, streaming=streaming + ) + + assert content == expected_content + assert len(tool_calls) == 1 + assert tool_calls[0].function == SIMPLE_FUNCTION_CALL + + +@pytest.mark.parametrize("streaming", [True, False]) +def test_leading_underscore_identifiers( + streaming: bool, default_tokenizer: TokenizerLike +): + """Leading-underscore names are valid Python and JSON Schema names.""" + tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( + default_tokenizer + ) + + content, tool_calls = run_tool_extraction( + tool_parser, "[_lookup(_id='abc', limit=5)]", streaming=streaming + ) + + assert content is None + assert len(tool_calls) == 1 + assert tool_calls[0].function == FunctionCall( + name="_lookup", arguments='{"_id": "abc", "limit": 5}' + ) diff --git a/vllm/tool_parsers/llama4_pythonic_tool_parser.py b/vllm/tool_parsers/llama4_pythonic_tool_parser.py index fdffb48ff7fe..6179843c362a 100644 --- a/vllm/tool_parsers/llama4_pythonic_tool_parser.py +++ b/vllm/tool_parsers/llama4_pythonic_tool_parser.py @@ -25,6 +25,8 @@ compute_tool_delta, handle_single_tool, make_valid_python, + partial_tag_overlap, + split_pythonic_tool_calls, ) logger = init_logger(__name__) @@ -43,7 +45,7 @@ class Llama4PythonicToolParser(ToolParser): # Llama3.2 models more reliable. TOOL_CALL_REGEX = re.compile( - r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]", + r"\[([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s)?\),\s*)*([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s*)?\)\s*)+\]", re.DOTALL, ) @@ -53,6 +55,7 @@ def __init__( tools: list[Tool] | None = None, ): super().__init__(tokenizer, tools) + self._streamed_trailing_text = "" # Rename for readability. This is NOT a tool id. @property @@ -73,11 +76,15 @@ def extract_tool_calls( model_output = model_output[len("<|python_start|>") :] model_output = model_output.replace("<|python_end|>", "") + split = split_pythonic_tool_calls(model_output) + tool_call_text, trailing_text = split if split else (model_output, "") + is_tool_call_pattern = False try: is_tool_call_pattern = ( self.TOOL_CALL_REGEX.match( - model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS + tool_call_text, + timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS, ) is not None ) @@ -93,7 +100,7 @@ def extract_tool_calls( ) try: - module = ast.parse(model_output) + module = ast.parse(tool_call_text) parsed = getattr(module.body[0], "value", None) if isinstance(parsed, ast.List) and all( isinstance(e, ast.Call) for e in parsed.elts @@ -104,7 +111,7 @@ def extract_tool_calls( handle_single_tool(e) # type: ignore for e in parsed.elts ], - content=None, + content=trailing_text.lstrip() or None, ) else: raise UnexpectedAstError("Tool output must be a list of function calls") @@ -136,7 +143,14 @@ def extract_tool_calls_streaming( current_text = current_text[len("<|python_start|>") :] if current_text.endswith("<|python_end|>"): current_text = current_text[: current_text.rfind("<|python_end|>")] - valid_and_added_text = make_valid_python(current_text) + split = split_pythonic_tool_calls(current_text) + tool_call_text, trailing_text = split if split else (current_text, "") + trailing_text = trailing_text.replace("<|python_end|>", "") + trailing_text = trailing_text[ + : len(trailing_text) + - partial_tag_overlap(trailing_text, "<|python_end|>") + ] + valid_and_added_text = make_valid_python(tool_call_text) if valid_and_added_text is None: return None valid_text, added_text = valid_and_added_text @@ -194,8 +208,14 @@ def extract_tool_calls_streaming( if tool_deltas and not self.prev_tool_call_arr: self.prev_tool_call_arr = [{"arguments": {}}] + trailing_text = trailing_text.lstrip() + content = trailing_text[len(self._streamed_trailing_text) :] + self._streamed_trailing_text = trailing_text + if tool_deltas: - return DeltaMessage(tool_calls=tool_deltas) + return DeltaMessage(tool_calls=tool_deltas, content=content or None) + elif content: + return DeltaMessage(content=content) elif not added_text and self.current_tool_id > 0: # Return an empty DeltaMessage once the tool calls are all done # so that finish_reason gets set. diff --git a/vllm/tool_parsers/olmo3_tool_parser.py b/vllm/tool_parsers/olmo3_tool_parser.py index cb7bc632baa8..f71045499323 100644 --- a/vllm/tool_parsers/olmo3_tool_parser.py +++ b/vllm/tool_parsers/olmo3_tool_parser.py @@ -47,7 +47,7 @@ class Olmo3PythonicToolParser(ToolParser): # Llama3.2 models more reliable. TOOL_CALL_REGEX = re.compile( - r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]", + r"\[([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s)?\),\s*)*([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s*)?\)\s*)+\]", re.DOTALL, ) diff --git a/vllm/tool_parsers/pythonic_tool_parser.py b/vllm/tool_parsers/pythonic_tool_parser.py index 76b263cfc5bb..37581d8570f9 100644 --- a/vllm/tool_parsers/pythonic_tool_parser.py +++ b/vllm/tool_parsers/pythonic_tool_parser.py @@ -25,6 +25,7 @@ compute_tool_delta, handle_single_tool, make_valid_python, + split_pythonic_tool_calls, ) logger = init_logger(__name__) @@ -45,7 +46,7 @@ class PythonicToolParser(ToolParser): # Llama3.2 models more reliable. TOOL_CALL_REGEX = re.compile( - r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]", + r"\[([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s)?\),\s*)*([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s*)?\)\s*)+\]", re.DOTALL, ) @@ -55,6 +56,7 @@ def __init__( tools: list[Tool] | None = None, ): super().__init__(tokenizer, tools) + self._streamed_trailing_text = "" # Rename for readability. This is NOT a tool id. @property @@ -69,11 +71,15 @@ def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: """Extract the tool calls from a complete model response.""" + split = split_pythonic_tool_calls(model_output) + tool_call_text, trailing_text = split if split else (model_output, "") + is_tool_call_pattern = False try: is_tool_call_pattern = ( self.TOOL_CALL_REGEX.match( - model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS + tool_call_text, + timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS, ) is not None ) @@ -89,7 +95,7 @@ def extract_tool_calls( ) try: - module = ast.parse(model_output) + module = ast.parse(tool_call_text) parsed = getattr(module.body[0], "value", None) if isinstance(parsed, ast.List) and all( isinstance(e, ast.Call) for e in parsed.elts @@ -100,7 +106,7 @@ def extract_tool_calls( handle_single_tool(e) # type: ignore for e in parsed.elts ], - content=None, + content=trailing_text.lstrip() or None, ) else: raise UnexpectedAstError("Tool output must be a list of function calls") @@ -125,7 +131,9 @@ def extract_tool_calls_streaming( return DeltaMessage(content=delta_text) try: - valid_and_added_text = make_valid_python(current_text) + split = split_pythonic_tool_calls(current_text) + tool_call_text, trailing_text = split if split else (current_text, "") + valid_and_added_text = make_valid_python(tool_call_text) if valid_and_added_text is None: return None valid_text, added_text = valid_and_added_text @@ -183,8 +191,14 @@ def extract_tool_calls_streaming( if tool_deltas and not self.prev_tool_call_arr: self.prev_tool_call_arr = [{"arguments": {}}] + trailing_text = trailing_text.lstrip() + content = trailing_text[len(self._streamed_trailing_text) :] + self._streamed_trailing_text = trailing_text + if tool_deltas: - return DeltaMessage(tool_calls=tool_deltas) + return DeltaMessage(tool_calls=tool_deltas, content=content or None) + elif content: + return DeltaMessage(content=content) elif not added_text and self.current_tool_id > 0: # Return an empty DeltaMessage once the tool calls are all done # so that finish_reason gets set. diff --git a/vllm/tool_parsers/utils.py b/vllm/tool_parsers/utils.py index d2c8c6f41220..d9e52c1a5d0c 100644 --- a/vllm/tool_parsers/utils.py +++ b/vllm/tool_parsers/utils.py @@ -907,6 +907,30 @@ def _is_escaped(text: str, index: int) -> bool: return backslashes % 2 == 1 +def split_pythonic_tool_calls(text: str) -> tuple[str, str] | None: + """Split ``[call(...), ...]`` from any text that follows it. + + Returns ``(call_list, trailing_text)`` once the list opened by the leading + ``[`` has closed, or None if the text does not start with ``[`` or the list + is still open. Brackets inside string literals are ignored. + """ + if not text.startswith("["): + return None + bracket_stack: list[str] = [] + for index, char in enumerate(text): + if bracket_stack and bracket_stack[-1] in {"'", '"'}: + if char == bracket_stack[-1] and not _is_escaped(text, index): + bracket_stack.pop() + continue + if char in {"[", "(", "{", "'", '"'}: + bracket_stack.append(char) + elif char in {"]", ")", "}"}: + bracket_stack.pop() + if not bracket_stack: + return text[: index + 1], text[index + 1 :] + return None + + def make_valid_python(text: str) -> tuple[str, str] | None: """Attempt to close all open brackets/quotes to make partial Python valid.