From f9f8e83202a65e90e051d129a919a56ab013ac6d Mon Sep 17 00:00:00 2001
From: RickyYii <237135932+RickyYii@users.noreply.github.com>
Date: Tue, 8 Sep 2026 01:39:03 +0000
Subject: [PATCH] [Bugfix][Tool Parser] Ignore inside JSON strings
in Hermes parser
The Hermes parser located the end of a tool call with a non-greedy regex
(non-streaming) and a plain `str.find` (streaming), so a literal
`` inside a JSON string argument terminated the tool call early
and the whole call was dropped with a JSONDecodeError. Scan the JSON body
with string/escape awareness instead, matching what the Rust frontend's
Hermes parser already does, and only withhold a partial closing tag in
streaming when it is not inside a string literal.
Fixes #45167
Signed-off-by: RickyYii <237135932+RickyYii@users.noreply.github.com>
---
tests/tool_parsers/test_hermes_tool_parser.py | 124 ++++++++++++++++++
vllm/tool_parsers/hermes_tool_parser.py | 75 ++++++-----
vllm/tool_parsers/longcat_tool_parser.py | 8 --
vllm/tool_parsers/utils.py | 40 ++++++
4 files changed, 207 insertions(+), 40 deletions(-)
diff --git a/tests/tool_parsers/test_hermes_tool_parser.py b/tests/tool_parsers/test_hermes_tool_parser.py
index 25869777dba5..13d7b598a711 100644
--- a/tests/tool_parsers/test_hermes_tool_parser.py
+++ b/tests/tool_parsers/test_hermes_tool_parser.py
@@ -412,3 +412,127 @@ def test_hermes_streaming_content_and_tool_call_in_single_chunk(
assert tool_parts[0].function.name == "f"
args_str = "".join(tc.function.arguments or "" for tc in tool_parts)
assert json.loads(args_str) == {"x": 1}
+
+
+# Reproduction from https://github.com/vllm-project/vllm/issues/45167
+END_TAG_IN_STRING_TEXT = (
+ "\n"
+ '{"name": "edit_file", "arguments": '
+ '{"content": "예시 태그: 포함 한국어 본문"}}\n'
+ ""
+)
+
+
+def test_hermes_parser_non_streaming_end_tag_inside_string(
+ qwen_tokenizer: TokenizerLike,
+ any_chat_request: ChatCompletionRequest,
+) -> None:
+ """A literal in a string argument must not end the tool call."""
+ parser = Hermes2ProToolParser(qwen_tokenizer)
+ tool_call = parser.extract_tool_calls(
+ model_output=END_TAG_IN_STRING_TEXT,
+ request=any_chat_request,
+ )
+
+ assert tool_call.tools_called
+ assert tool_call.content is None
+ assert len(tool_call.tool_calls) == 1
+ assert tool_call.tool_calls[0].function.name == "edit_file"
+ args = json.loads(tool_call.tool_calls[0].function.arguments)
+ assert args["content"] == "예시 태그: 포함 한국어 본문"
+
+
+def test_hermes_parser_non_streaming_end_tag_inside_first_of_two_calls(
+ qwen_tokenizer: TokenizerLike,
+ any_chat_request: ChatCompletionRequest,
+) -> None:
+ """A quoted end tag must not swallow the tool call that follows it."""
+ text = (
+ '{"name": "first", '
+ '"arguments": {"q": "a b"}}'
+ '{"name": "second", "arguments": {"q": "c"}}'
+ )
+ parser = Hermes2ProToolParser(qwen_tokenizer)
+ tool_call = parser.extract_tool_calls(
+ model_output=text,
+ request=any_chat_request,
+ )
+
+ assert tool_call.tools_called
+ assert [tc.function.name for tc in tool_call.tool_calls] == ["first", "second"]
+ assert json.loads(tool_call.tool_calls[0].function.arguments) == {
+ "q": "a b"
+ }
+ assert json.loads(tool_call.tool_calls[1].function.arguments) == {"q": "c"}
+
+
+def test_hermes_parser_non_streaming_escaped_quote_before_end_tag(
+ qwen_tokenizer: TokenizerLike,
+ any_chat_request: ChatCompletionRequest,
+) -> None:
+ """Escaped quotes must not confuse the string tracking of the scanner."""
+ text = (
+ ""
+ '{"name": "say", "arguments": {"text": "say \\"hi\\" done"}}'
+ ""
+ )
+ parser = Hermes2ProToolParser(qwen_tokenizer)
+ tool_call = parser.extract_tool_calls(
+ model_output=text,
+ request=any_chat_request,
+ )
+
+ assert tool_call.tools_called
+ assert tool_call.tool_calls[0].function.name == "say"
+ args = json.loads(tool_call.tool_calls[0].function.arguments)
+ assert args["text"] == 'say "hi" done'
+
+
+def test_hermes_parser_streaming_end_tag_inside_string(
+ qwen_tokenizer: TokenizerLike,
+ any_chat_request: ChatCompletionRequest,
+) -> None:
+ """Streaming a quoted end tag must yield the non-streaming tool call."""
+ parser = Hermes2ProToolParser(qwen_tokenizer)
+ deltas = _simulate_streaming(
+ qwen_tokenizer, parser, any_chat_request, END_TAG_IN_STRING_TEXT
+ )
+
+ tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
+ assert {tc.index for tc in tool_calls} == {0}
+ assert tool_calls[0].function.name == "edit_file"
+ args = json.loads("".join(tc.function.arguments or "" for tc in tool_calls))
+ assert "" in args["content"]
+
+ # Compare against the text the deltas reconstruct: decoding one token at a
+ # time is lossy for multi-byte characters.
+ streamed_text = "".join(
+ qwen_tokenizer.decode([token])
+ for token in qwen_tokenizer.encode(END_TAG_IN_STRING_TEXT)
+ )
+ expected = Hermes2ProToolParser(qwen_tokenizer).extract_tool_calls(
+ model_output=streamed_text,
+ request=any_chat_request,
+ )
+ assert args == json.loads(expected.tool_calls[0].function.arguments)
+
+
+def test_hermes_parser_streaming_start_tag_inside_string(
+ qwen_tokenizer: TokenizerLike,
+ any_chat_request: ChatCompletionRequest,
+) -> None:
+ """A quoted start tag must not open a second tool call while streaming."""
+ text = (
+ ""
+ '{"name": "edit_file", "arguments": '
+ '{"content": "opens and closes "}}'
+ ""
+ )
+ parser = Hermes2ProToolParser(qwen_tokenizer)
+ deltas = _simulate_streaming(qwen_tokenizer, parser, any_chat_request, text)
+
+ tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls]
+ assert {tc.index for tc in tool_calls} == {0}
+ assert tool_calls[0].function.name == "edit_file"
+ args = json.loads("".join(tc.function.arguments or "" for tc in tool_calls))
+ assert args["content"] == "opens and closes "
diff --git a/vllm/tool_parsers/hermes_tool_parser.py b/vllm/tool_parsers/hermes_tool_parser.py
index c5b318cd19fc..dda736f3557f 100644
--- a/vllm/tool_parsers/hermes_tool_parser.py
+++ b/vllm/tool_parsers/hermes_tool_parser.py
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
-from collections.abc import Sequence
+from collections.abc import Iterator, Sequence
import regex as re
@@ -25,7 +25,11 @@
Tool,
ToolParser,
)
-from vllm.tool_parsers.utils import is_complete_json, partial_tag_overlap
+from vllm.tool_parsers.utils import (
+ find_tag_outside_json_strings,
+ is_complete_json,
+ partial_tag_overlap,
+)
from vllm.utils.mistral import is_mistral_tokenizer
logger = init_logger(__name__)
@@ -35,9 +39,6 @@ class Hermes2ProToolParser(ToolParser):
structural_tag_model = "hermes"
tool_call_start_token: str = ""
tool_call_end_token: str = ""
- tool_call_regex = re.compile(
- r"(.*?)|(.*)", re.DOTALL
- )
scratch_pad_regex = re.compile(r"(.*?)", re.DOTALL)
def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
@@ -67,6 +68,29 @@ def adjust_request(
request.skip_special_tokens = False
return request
+ def _iter_tool_call_bodies(self, text: str) -> Iterator[tuple[str, bool, bool]]:
+ """Yield the raw body of each tool call region in ``text``.
+
+ The end token is located with JSON string awareness so that a literal
+ ```` inside a string argument does not end the region. A
+ region without an end token extends to the end of ``text``.
+
+ Yields:
+ The body, whether the end token was found, and whether the body
+ ends inside an unterminated JSON string literal.
+ """
+ pos = 0
+ while (start := text.find(self.tool_call_start_token, pos)) != -1:
+ body_start = start + len(self.tool_call_start_token)
+ end, in_string = find_tag_outside_json_strings(
+ text, self.tool_call_end_token, body_start
+ )
+ if end == -1:
+ yield text[body_start:], False, in_string
+ return
+ yield text[body_start:end], True, False
+ pos = end + len(self.tool_call_end_token)
+
def extract_tool_calls(
self,
model_output: str,
@@ -80,17 +104,11 @@ def extract_tool_calls(
else:
try:
- # there are two possible captures - between tags, or between a
- # tag and end-of-string so the result of
- # findall is an array of tuples where one is a function call and
- # the other is None
- function_call_tuples = self.tool_call_regex.findall(model_output)
-
# load the JSON, and then use it to build the Function and
# Tool Call
raw_function_calls = [
- json.loads(match[0] if match[0] else match[1])
- for match in function_call_tuples
+ json.loads(body)
+ for body, _, _ in self._iter_tool_call_bodies(model_output)
]
tool_calls = [
ToolCall(
@@ -141,28 +159,21 @@ def _extract_content(self, current_text: str) -> str | None:
def _extract_tool_call_jsons(self, text: str) -> list[tuple[str, bool]]:
"""Extract (json_text, is_complete) for each region."""
results: list[tuple[str, bool]] = []
- pos = 0
- while True:
- start = text.find(self.tool_call_start_token, pos)
- if start == -1:
- break
- json_start = start + len(self.tool_call_start_token)
- json_end = text.find(self.tool_call_end_token, json_start)
- if json_end != -1:
- results.append((text[json_start:json_end].strip(), True))
- pos = json_end + len(self.tool_call_end_token)
- else:
- raw = text[json_start:]
- # Strip partial suffix if present.
+ for raw, terminated, in_string in self._iter_tool_call_bodies(text):
+ if terminated:
+ results.append((raw.strip(), True))
+ continue
+ if not in_string:
+ # Strip partial suffix if present. Inside a
+ # string literal those characters are argument data.
overlap = partial_tag_overlap(raw, self.tool_call_end_token)
if overlap:
raw = raw[:-overlap]
- tc_json = raw.strip()
- # Valid JSON without closing tag = complete body,
- # tag tokens just haven't arrived yet.
- is_complete = is_complete_json(tc_json) if tc_json else False
- results.append((tc_json, is_complete))
- break
+ tc_json = raw.strip()
+ # Valid JSON without closing tag = complete body,
+ # tag tokens just haven't arrived yet.
+ is_complete = is_complete_json(tc_json) if tc_json else False
+ results.append((tc_json, is_complete))
return results
@staticmethod
diff --git a/vllm/tool_parsers/longcat_tool_parser.py b/vllm/tool_parsers/longcat_tool_parser.py
index ccfa94167256..81d6e2d1a805 100644
--- a/vllm/tool_parsers/longcat_tool_parser.py
+++ b/vllm/tool_parsers/longcat_tool_parser.py
@@ -1,8 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-import regex as re
-
from vllm.tokenizers import TokenizerLike
from vllm.tool_parsers.abstract_tool_parser import Tool
from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser
@@ -14,9 +12,3 @@ def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
self.tool_call_start_token: str = ""
self.tool_call_end_token: str = ""
-
- self.tool_call_regex = re.compile(
- r"(.*?)"
- r"|(.*)",
- re.DOTALL,
- )
diff --git a/vllm/tool_parsers/utils.py b/vllm/tool_parsers/utils.py
index d39c83969f41..52653a1ca3c6 100644
--- a/vllm/tool_parsers/utils.py
+++ b/vllm/tool_parsers/utils.py
@@ -55,6 +55,46 @@ def partial_tag_overlap(text: str, tag: str) -> int:
return 0
+def find_tag_outside_json_strings(
+ text: str, tag: str, start: int = 0
+) -> tuple[int, bool]:
+ """Find a tag that is not part of a JSON string literal.
+
+ Scanning starts outside a string and tracks JSON string and escape state,
+ so a tag inside a string value is argument data rather than markup.
+
+ Args:
+ text: The text to scan.
+ tag: The tag to look for, e.g. ``""``.
+ start: Index in ``text`` to start scanning from.
+
+ Returns:
+ The index of the first such tag at or after ``start``, or -1 if there
+ is none, together with whether the scan reached the end of ``text``
+ inside an unterminated string literal.
+ """
+ pos = start
+ in_string = False
+ tag_pos = text.find(tag, start)
+ while pos < len(text):
+ quote = text.find('"', pos)
+ if in_string:
+ if quote == -1:
+ return -1, True
+ in_string = _is_escaped(text, quote)
+ pos = quote + 1
+ continue
+ if 0 <= tag_pos < pos:
+ tag_pos = text.find(tag, pos)
+ if tag_pos != -1 and (quote == -1 or tag_pos < quote):
+ return tag_pos, False
+ if quote == -1:
+ return -1, False
+ in_string = True
+ pos = quote + 1
+ return -1, in_string
+
+
def find_common_prefix(s1: str, s2: str) -> str:
"""
Finds a common prefix that is shared between two strings, if there is one.