[Bugfix][Tool Parser] Ignore </tool_call> inside JSON strings in Hermes parser

tests/tool_parsers/test_hermes_tool_parser.py

@@ -412,3 +412,127 @@ def test_hermes_streaming_content_and_tool_call_in_single_chunk(
     assert tool_parts[0].function.name == "f"
     args_str = "".join(tc.function.arguments or "" for tc in tool_parts)
     assert json.loads(args_str) == {"x": 1}
+
+
+# Reproduction from https://github.com/vllm-project/vllm/issues/45167
+END_TAG_IN_STRING_TEXT = (
+    "<tool_call>\n"
+    '{"name": "edit_file", "arguments": '
+    '{"content": "예시 태그: </tool_call> 포함 한국어 본문"}}\n'
+    "</tool_call>"
+)
+
+
+def test_hermes_parser_non_streaming_end_tag_inside_string(
+    qwen_tokenizer: TokenizerLike,
+    any_chat_request: ChatCompletionRequest,
+) -> None:
+    """A literal </tool_call> in a string argument must not end the tool call."""
+    parser = Hermes2ProToolParser(qwen_tokenizer)
+    tool_call = parser.extract_tool_calls(
+        model_output=END_TAG_IN_STRING_TEXT,
+        request=any_chat_request,
+    )
+
+    assert tool_call.tools_called
+    assert tool_call.content is None
+    assert len(tool_call.tool_calls) == 1
+    assert tool_call.tool_calls[0].function.name == "edit_file"
+    args = json.loads(tool_call.tool_calls[0].function.arguments)
+    assert args["content"] == "예시 태그: </tool_call> 포함 한국어 본문"
+
+
+def test_hermes_parser_non_streaming_end_tag_inside_first_of_two_calls(
+    qwen_tokenizer: TokenizerLike,
+    any_chat_request: ChatCompletionRequest,
+) -> None:
+    """A quoted end tag must not swallow the tool call that follows it."""
+    text = (
+        '<tool_call>{"name": "first", '
+        '"arguments": {"q": "a </tool_call> b"}}</tool_call>'
+        '<tool_call>{"name": "second", "arguments": {"q": "c"}}</tool_call>'
+    )
+    parser = Hermes2ProToolParser(qwen_tokenizer)
+    tool_call = parser.extract_tool_calls(
+        model_output=text,
+        request=any_chat_request,
+    )
+
+    assert tool_call.tools_called
+    assert [tc.function.name for tc in tool_call.tool_calls] == ["first", "second"]
+    assert json.loads(tool_call.tool_calls[0].function.arguments) == {
+        "q": "a </tool_call> b"
+    }
+    assert json.loads(tool_call.tool_calls[1].function.arguments) == {"q": "c"}
+
+
+def test_hermes_parser_non_streaming_escaped_quote_before_end_tag(
+    qwen_tokenizer: TokenizerLike,
+    any_chat_request: ChatCompletionRequest,
+) -> None:
+    """Escaped quotes must not confuse the string tracking of the scanner."""
+    text = (
+        "<tool_call>"
+        '{"name": "say", "arguments": {"text": "say \\"hi\\" </tool_call> done"}}'
+        "</tool_call>"
+    )
+    parser = Hermes2ProToolParser(qwen_tokenizer)
+    tool_call = parser.extract_tool_calls(
+        model_output=text,
+        request=any_chat_request,
+    )
+
+    assert tool_call.tools_called
+    assert tool_call.tool_calls[0].function.name == "say"
+    args = json.loads(tool_call.tool_calls[0].function.arguments)
+    assert args["text"] == 'say "hi" </tool_call> done'
+
+
+def test_hermes_parser_streaming_end_tag_inside_string(
+    qwen_tokenizer: TokenizerLike,
+    any_chat_request: ChatCompletionRequest,
+) -> None:
+    """Streaming a quoted end tag must yield the non-streaming tool call."""
+    parser = Hermes2ProToolParser(qwen_tokenizer)
+    deltas = _simulate_streaming(
+        qwen_tokenizer, parser, any_chat_request, END_TAG_IN_STRING_TEXT
+    )
+
+    tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
+    assert {tc.index for tc in tool_calls} == {0}
+    assert tool_calls[0].function.name == "edit_file"
+    args = json.loads("".join(tc.function.arguments or "" for tc in tool_calls))
+    assert "</tool_call>" in args["content"]
+
+    # Compare against the text the deltas reconstruct: decoding one token at a
+    # time is lossy for multi-byte characters.
+    streamed_text = "".join(
+        qwen_tokenizer.decode([token])
+        for token in qwen_tokenizer.encode(END_TAG_IN_STRING_TEXT)
+    )
+    expected = Hermes2ProToolParser(qwen_tokenizer).extract_tool_calls(
+        model_output=streamed_text,
+        request=any_chat_request,
+    )
+    assert args == json.loads(expected.tool_calls[0].function.arguments)
+
+
+def test_hermes_parser_streaming_start_tag_inside_string(
+    qwen_tokenizer: TokenizerLike,
+    any_chat_request: ChatCompletionRequest,
+) -> None:
+    """A quoted start tag must not open a second tool call while streaming."""
+    text = (
+        "<tool_call>"
+        '{"name": "edit_file", "arguments": '
+        '{"content": "opens <tool_call> and closes </tool_call>"}}'
+        "</tool_call>"
+    )
+    parser = Hermes2ProToolParser(qwen_tokenizer)
+    deltas = _simulate_streaming(qwen_tokenizer, parser, any_chat_request, text)
+
+    tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
+    assert {tc.index for tc in tool_calls} == {0}
+    assert tool_calls[0].function.name == "edit_file"
+    args = json.loads("".join(tc.function.arguments or "" for tc in tool_calls))
+    assert args["content"] == "opens <tool_call> and closes </tool_call>"

vllm/tool_parsers/hermes_tool_parser.py

@@ -2,7 +2,7 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import json
-from collections.abc import Sequence
+from collections.abc import Iterator, Sequence
 
 import regex as re
 
@@ -25,7 +25,11 @@
     Tool,
     ToolParser,
 )
-from vllm.tool_parsers.utils import is_complete_json, partial_tag_overlap
+from vllm.tool_parsers.utils import (
+    find_tag_outside_json_strings,
+    is_complete_json,
+    partial_tag_overlap,
+)
 from vllm.utils.mistral import is_mistral_tokenizer
 
 logger = init_logger(__name__)
@@ -35,9 +39,6 @@ class Hermes2ProToolParser(ToolParser):
     structural_tag_model = "hermes"
     tool_call_start_token: str = "<tool_call>"
     tool_call_end_token: str = "</tool_call>"
-    tool_call_regex = re.compile(
-        r"<tool_call>(.*?)</tool_call>|<tool_call>(.*)", re.DOTALL
-    )
     scratch_pad_regex = re.compile(r"<scratch_pad>(.*?)</scratch_pad>", re.DOTALL)
 
     def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
@@ -67,6 +68,29 @@ def adjust_request(
             request.skip_special_tokens = False
         return request
 
+    def _iter_tool_call_bodies(self, text: str) -> Iterator[tuple[str, bool, bool]]:
+        """Yield the raw body of each tool call region in ``text``.
+
+        The end token is located with JSON string awareness so that a literal
+        ``</tool_call>`` inside a string argument does not end the region. A
+        region without an end token extends to the end of ``text``.
+
+        Yields:
+            The body, whether the end token was found, and whether the body
+            ends inside an unterminated JSON string literal.
+        """
+        pos = 0
+        while (start := text.find(self.tool_call_start_token, pos)) != -1:
+            body_start = start + len(self.tool_call_start_token)
+            end, in_string = find_tag_outside_json_strings(
+                text, self.tool_call_end_token, body_start
+            )
+            if end == -1:
+                yield text[body_start:], False, in_string
+                return
+            yield text[body_start:end], True, False
+            pos = end + len(self.tool_call_end_token)
+
     def extract_tool_calls(
         self,
         model_output: str,
@@ -80,17 +104,11 @@ def extract_tool_calls(
 
         else:
             try:
-                # there are two possible captures - between tags, or between a
-                # tag and end-of-string so the result of
-                # findall is an array of tuples where one is a function call and
-                # the other is None
-                function_call_tuples = self.tool_call_regex.findall(model_output)
-
                 # load the JSON, and then use it to build the Function and
                 # Tool Call
                 raw_function_calls = [
-                    json.loads(match[0] if match[0] else match[1])
-                    for match in function_call_tuples
+                    json.loads(body)
+                    for body, _, _ in self._iter_tool_call_bodies(model_output)
                 ]
                 tool_calls = [
                     ToolCall(
@@ -141,28 +159,21 @@ def _extract_content(self, current_text: str) -> str | None:
     def _extract_tool_call_jsons(self, text: str) -> list[tuple[str, bool]]:
         """Extract (json_text, is_complete) for each <tool_call> region."""
         results: list[tuple[str, bool]] = []
-        pos = 0
-        while True:
-            start = text.find(self.tool_call_start_token, pos)
-            if start == -1:
-                break
-            json_start = start + len(self.tool_call_start_token)
-            json_end = text.find(self.tool_call_end_token, json_start)
-            if json_end != -1:
-                results.append((text[json_start:json_end].strip(), True))
-                pos = json_end + len(self.tool_call_end_token)
-            else:
-                raw = text[json_start:]
-                # Strip partial </tool_call> suffix if present.
+        for raw, terminated, in_string in self._iter_tool_call_bodies(text):
+            if terminated:
+                results.append((raw.strip(), True))
+                continue
+            if not in_string:
+                # Strip partial </tool_call> suffix if present. Inside a
+                # string literal those characters are argument data.
                 overlap = partial_tag_overlap(raw, self.tool_call_end_token)
                 if overlap:
                     raw = raw[:-overlap]
-                tc_json = raw.strip()
-                # Valid JSON without closing tag = complete body,
-                # tag tokens just haven't arrived yet.
-                is_complete = is_complete_json(tc_json) if tc_json else False
-                results.append((tc_json, is_complete))
-                break
+            tc_json = raw.strip()
+            # Valid JSON without closing tag = complete body,
+            # tag tokens just haven't arrived yet.
+            is_complete = is_complete_json(tc_json) if tc_json else False
+            results.append((tc_json, is_complete))
         return results
 
     @staticmethod

vllm/tool_parsers/longcat_tool_parser.py

@@ -1,8 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-import regex as re
-
 from vllm.tokenizers import TokenizerLike
 from vllm.tool_parsers.abstract_tool_parser import Tool
 from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
@@ -14,9 +12,3 @@ def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
 
         self.tool_call_start_token: str = "<longcat_tool_call>"
         self.tool_call_end_token: str = "</longcat_tool_call>"
-
-        self.tool_call_regex = re.compile(
-            r"<longcat_tool_call>(.*?)</longcat_tool_call>"
-            r"|<longcat_tool_call>(.*)",
-            re.DOTALL,
-        )

vllm/tool_parsers/utils.py

@@ -55,6 +55,46 @@ def partial_tag_overlap(text: str, tag: str) -> int:
     return 0
 
 
+def find_tag_outside_json_strings(
+    text: str, tag: str, start: int = 0
+) -> tuple[int, bool]:
+    """Find a tag that is not part of a JSON string literal.
+
+    Scanning starts outside a string and tracks JSON string and escape state,
+    so a tag inside a string value is argument data rather than markup.
+
+    Args:
+        text: The text to scan.
+        tag: The tag to look for, e.g. ``"</tool_call>"``.
+        start: Index in ``text`` to start scanning from.
+
+    Returns:
+        The index of the first such tag at or after ``start``, or -1 if there
+        is none, together with whether the scan reached the end of ``text``
+        inside an unterminated string literal.
+    """
+    pos = start
+    in_string = False
+    tag_pos = text.find(tag, start)
+    while pos < len(text):
+        quote = text.find('"', pos)
+        if in_string:
+            if quote == -1:
+                return -1, True
+            in_string = _is_escaped(text, quote)
+            pos = quote + 1
+            continue
+        if 0 <= tag_pos < pos:
+            tag_pos = text.find(tag, pos)
+        if tag_pos != -1 and (quote == -1 or tag_pos < quote):
+            return tag_pos, False
+        if quote == -1:
+            return -1, False
+        in_string = True
+        pos = quote + 1
+    return -1, in_string
+
+
 def find_common_prefix(s1: str, s2: str) -> str:
     """
     Finds a common prefix that is shared between two strings, if there is one.