[Bugfix][Tool Parser] Make pythonic tool parsers agree across streaming modes

tests/tool_parsers/test_llama4_pythonic_tool_parser.py

@@ -267,3 +267,59 @@ def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLik
         assert content == fake_problematic_input
         assert len(tool_calls) == 0
         mock_regex.match.assert_called_once()
+
+
+TRAILING_TEXT_CASES = [
+    pytest.param(
+        f"{SIMPLE_FUNCTION_OUTPUT} Let me check that for you.",
+        "Let me check that for you.",
+        id="same_line",
+    ),
+    pytest.param(
+        f"{SIMPLE_FUNCTION_OUTPUT}\n\nI'll wait for the result.",
+        "I'll wait for the result.",
+        id="after_blank_line",
+    ),
+]
+
+
+@pytest.mark.parametrize("streaming", [True, False])
+@pytest.mark.parametrize("model_output, expected_content", TRAILING_TEXT_CASES)
+def test_tool_call_followed_by_text(
+    streaming: bool,
+    model_output: str,
+    expected_content: str,
+    default_tokenizer: TokenizerLike,
+):
+    """Both modes return the call and the text after it."""
+    tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
+        default_tokenizer
+    )
+
+    content, tool_calls = run_tool_extraction(
+        tool_parser, model_output, streaming=streaming
+    )
+
+    assert content == expected_content
+    assert len(tool_calls) == 1
+    assert tool_calls[0].function == SIMPLE_FUNCTION_CALL
+
+
+@pytest.mark.parametrize("streaming", [True, False])
+def test_leading_underscore_identifiers(
+    streaming: bool, default_tokenizer: TokenizerLike
+):
+    """Leading-underscore names are valid Python and JSON Schema names."""
+    tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")(
+        default_tokenizer
+    )
+
+    content, tool_calls = run_tool_extraction(
+        tool_parser, "[_lookup(_id='abc', limit=5)]", streaming=streaming
+    )
+
+    assert content is None
+    assert len(tool_calls) == 1
+    assert tool_calls[0].function == FunctionCall(
+        name="_lookup", arguments='{"_id": "abc", "limit": 5}'
+    )

tests/tool_parsers/test_olmo3_tool_parser.py

@@ -249,3 +249,25 @@ def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLik
         assert content == fake_problematic_input
         assert len(tool_calls) == 0
         mock_regex.match.assert_called_once()
+
+
+@pytest.mark.parametrize("streaming", [True, False])
+def test_leading_underscore_identifiers(
+    streaming: bool, default_tokenizer: TokenizerLike
+):
+    """Leading-underscore names are valid Python and JSON Schema names."""
+    tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")(
+        default_tokenizer
+    )
+
+    content, tool_calls = run_tool_extraction(
+        tool_parser,
+        "<function_calls>_lookup(_id='abc', limit=5)</function_calls>",
+        streaming=streaming,
+    )
+
+    assert content is None
+    assert len(tool_calls) == 1
+    assert tool_calls[0].function == FunctionCall(
+        name="_lookup", arguments='{"_id": "abc", "limit": 5}'
+    )

tests/tool_parsers/test_pythonic_tool_parser.py

@@ -229,3 +229,59 @@ def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLik
         assert content == fake_problematic_input
         assert len(tool_calls) == 0
         mock_regex.match.assert_called_once()
+
+
+TRAILING_TEXT_CASES = [
+    pytest.param(
+        f"[{SIMPLE_FUNCTION_OUTPUT}] Let me check that for you.",
+        "Let me check that for you.",
+        id="same_line",
+    ),
+    pytest.param(
+        f"[{SIMPLE_FUNCTION_OUTPUT}]\n\nI'll wait for the result.",
+        "I'll wait for the result.",
+        id="after_blank_line",
+    ),
+]
+
+
+@pytest.mark.parametrize("streaming", [True, False])
+@pytest.mark.parametrize("model_output, expected_content", TRAILING_TEXT_CASES)
+def test_tool_call_followed_by_text(
+    streaming: bool,
+    model_output: str,
+    expected_content: str,
+    default_tokenizer: TokenizerLike,
+):
+    """Both modes return the call and the text after it."""
+    tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
+        default_tokenizer
+    )
+
+    content, tool_calls = run_tool_extraction(
+        tool_parser, model_output, streaming=streaming
+    )
+
+    assert content == expected_content
+    assert len(tool_calls) == 1
+    assert tool_calls[0].function == SIMPLE_FUNCTION_CALL
+
+
+@pytest.mark.parametrize("streaming", [True, False])
+def test_leading_underscore_identifiers(
+    streaming: bool, default_tokenizer: TokenizerLike
+):
+    """Leading-underscore names are valid Python and JSON Schema names."""
+    tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")(
+        default_tokenizer
+    )
+
+    content, tool_calls = run_tool_extraction(
+        tool_parser, "[_lookup(_id='abc', limit=5)]", streaming=streaming
+    )
+
+    assert content is None
+    assert len(tool_calls) == 1
+    assert tool_calls[0].function == FunctionCall(
+        name="_lookup", arguments='{"_id": "abc", "limit": 5}'
+    )

vllm/tool_parsers/llama4_pythonic_tool_parser.py

@@ -25,6 +25,8 @@
     compute_tool_delta,
     handle_single_tool,
     make_valid_python,
+    partial_tag_overlap,
+    split_pythonic_tool_calls,
 )
 
 logger = init_logger(__name__)
@@ -43,7 +45,7 @@ class Llama4PythonicToolParser(ToolParser):
     # Llama3.2 models more reliable.
 
     TOOL_CALL_REGEX = re.compile(
-        r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]",
+        r"\[([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s)?\),\s*)*([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s*)?\)\s*)+\]",
         re.DOTALL,
     )
 
@@ -53,6 +55,7 @@ def __init__(
         tools: list[Tool] | None = None,
     ):
         super().__init__(tokenizer, tools)
+        self._streamed_trailing_text = ""
 
     # Rename for readability. This is NOT a tool id.
     @property
@@ -73,11 +76,15 @@ def extract_tool_calls(
             model_output = model_output[len("<|python_start|>") :]
             model_output = model_output.replace("<|python_end|>", "")
 
+        split = split_pythonic_tool_calls(model_output)
+        tool_call_text, trailing_text = split if split else (model_output, "")
+
         is_tool_call_pattern = False
         try:
             is_tool_call_pattern = (
                 self.TOOL_CALL_REGEX.match(
-                    model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS
+                    tool_call_text,
+                    timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS,
                 )
                 is not None
             )
@@ -93,7 +100,7 @@ def extract_tool_calls(
             )
 
         try:
-            module = ast.parse(model_output)
+            module = ast.parse(tool_call_text)
             parsed = getattr(module.body[0], "value", None)
             if isinstance(parsed, ast.List) and all(
                 isinstance(e, ast.Call) for e in parsed.elts
@@ -104,7 +111,7 @@ def extract_tool_calls(
                         handle_single_tool(e)  # type: ignore
                         for e in parsed.elts
                     ],
-                    content=None,
+                    content=trailing_text.lstrip() or None,
                 )
             else:
                 raise UnexpectedAstError("Tool output must be a list of function calls")
@@ -136,7 +143,14 @@ def extract_tool_calls_streaming(
                 current_text = current_text[len("<|python_start|>") :]
             if current_text.endswith("<|python_end|>"):
                 current_text = current_text[: current_text.rfind("<|python_end|>")]
-            valid_and_added_text = make_valid_python(current_text)
+            split = split_pythonic_tool_calls(current_text)
+            tool_call_text, trailing_text = split if split else (current_text, "")
+            trailing_text = trailing_text.replace("<|python_end|>", "")
+            trailing_text = trailing_text[
+                : len(trailing_text)
+                - partial_tag_overlap(trailing_text, "<|python_end|>")
+            ]
+            valid_and_added_text = make_valid_python(tool_call_text)
             if valid_and_added_text is None:
                 return None
             valid_text, added_text = valid_and_added_text
@@ -194,8 +208,14 @@ def extract_tool_calls_streaming(
             if tool_deltas and not self.prev_tool_call_arr:
                 self.prev_tool_call_arr = [{"arguments": {}}]
 
+            trailing_text = trailing_text.lstrip()
+            content = trailing_text[len(self._streamed_trailing_text) :]
+            self._streamed_trailing_text = trailing_text
+
             if tool_deltas:
-                return DeltaMessage(tool_calls=tool_deltas)
+                return DeltaMessage(tool_calls=tool_deltas, content=content or None)
+            elif content:
+                return DeltaMessage(content=content)
             elif not added_text and self.current_tool_id > 0:
                 # Return an empty DeltaMessage once the tool calls are all done
                 # so that finish_reason gets set.

vllm/tool_parsers/olmo3_tool_parser.py

@@ -47,7 +47,7 @@ class Olmo3PythonicToolParser(ToolParser):
     # Llama3.2 models more reliable.
 
     TOOL_CALL_REGEX = re.compile(
-        r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]",
+        r"\[([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s)?\),\s*)*([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s*)?\)\s*)+\]",
         re.DOTALL,
     )
 

vllm/tool_parsers/pythonic_tool_parser.py

@@ -25,6 +25,7 @@
     compute_tool_delta,
     handle_single_tool,
     make_valid_python,
+    split_pythonic_tool_calls,
 )
 
 logger = init_logger(__name__)
@@ -45,7 +46,7 @@ class PythonicToolParser(ToolParser):
     # Llama3.2 models more reliable.
 
     TOOL_CALL_REGEX = re.compile(
-        r"\[([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s)?\),\s*)*([a-zA-Z]+\w*\(([a-zA-Z]+\w*=.*,\s*)*([a-zA-Z]+\w*=.*\s*)?\)\s*)+\]",
+        r"\[([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s)?\),\s*)*([a-zA-Z_]\w*\(([a-zA-Z_]\w*=.*,\s*)*([a-zA-Z_]\w*=.*\s*)?\)\s*)+\]",
         re.DOTALL,
     )
 
@@ -55,6 +56,7 @@ def __init__(
         tools: list[Tool] | None = None,
     ):
         super().__init__(tokenizer, tools)
+        self._streamed_trailing_text = ""
 
     # Rename for readability. This is NOT a tool id.
     @property
@@ -69,11 +71,15 @@ def extract_tool_calls(
         self, model_output: str, request: ChatCompletionRequest
     ) -> ExtractedToolCallInformation:
         """Extract the tool calls from a complete model response."""
+        split = split_pythonic_tool_calls(model_output)
+        tool_call_text, trailing_text = split if split else (model_output, "")
+
         is_tool_call_pattern = False
         try:
             is_tool_call_pattern = (
                 self.TOOL_CALL_REGEX.match(
-                    model_output, timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS
+                    tool_call_text,
+                    timeout=envs.VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS,
                 )
                 is not None
             )
@@ -89,7 +95,7 @@ def extract_tool_calls(
             )
 
         try:
-            module = ast.parse(model_output)
+            module = ast.parse(tool_call_text)
             parsed = getattr(module.body[0], "value", None)
             if isinstance(parsed, ast.List) and all(
                 isinstance(e, ast.Call) for e in parsed.elts
@@ -100,7 +106,7 @@ def extract_tool_calls(
                         handle_single_tool(e)  # type: ignore
                         for e in parsed.elts
                     ],
-                    content=None,
+                    content=trailing_text.lstrip() or None,
                 )
             else:
                 raise UnexpectedAstError("Tool output must be a list of function calls")
@@ -125,7 +131,9 @@ def extract_tool_calls_streaming(
             return DeltaMessage(content=delta_text)
 
         try:
-            valid_and_added_text = make_valid_python(current_text)
+            split = split_pythonic_tool_calls(current_text)
+            tool_call_text, trailing_text = split if split else (current_text, "")
+            valid_and_added_text = make_valid_python(tool_call_text)
             if valid_and_added_text is None:
                 return None
             valid_text, added_text = valid_and_added_text
@@ -183,8 +191,14 @@ def extract_tool_calls_streaming(
             if tool_deltas and not self.prev_tool_call_arr:
                 self.prev_tool_call_arr = [{"arguments": {}}]
 
+            trailing_text = trailing_text.lstrip()
+            content = trailing_text[len(self._streamed_trailing_text) :]
+            self._streamed_trailing_text = trailing_text
+
             if tool_deltas:
-                return DeltaMessage(tool_calls=tool_deltas)
+                return DeltaMessage(tool_calls=tool_deltas, content=content or None)
+            elif content:
+                return DeltaMessage(content=content)
             elif not added_text and self.current_tool_id > 0:
                 # Return an empty DeltaMessage once the tool calls are all done
                 # so that finish_reason gets set.

vllm/tool_parsers/utils.py

@@ -907,6 +907,30 @@ def _is_escaped(text: str, index: int) -> bool:
     return backslashes % 2 == 1
 
 
+def split_pythonic_tool_calls(text: str) -> tuple[str, str] | None:
+    """Split ``[call(...), ...]`` from any text that follows it.
+
+    Returns ``(call_list, trailing_text)`` once the list opened by the leading
+    ``[`` has closed, or None if the text does not start with ``[`` or the list
+    is still open. Brackets inside string literals are ignored.
+    """
+    if not text.startswith("["):
+        return None
+    bracket_stack: list[str] = []
+    for index, char in enumerate(text):
+        if bracket_stack and bracket_stack[-1] in {"'", '"'}:
+            if char == bracket_stack[-1] and not _is_escaped(text, index):
+                bracket_stack.pop()
+            continue
+        if char in {"[", "(", "{", "'", '"'}:
+            bracket_stack.append(char)
+        elif char in {"]", ")", "}"}:
+            bracket_stack.pop()
+            if not bracket_stack:
+                return text[: index + 1], text[index + 1 :]
+    return None
+
+
 def make_valid_python(text: str) -> tuple[str, str] | None:
     """Attempt to close all open brackets/quotes to make partial Python valid.