diff --git a/tests/tool_parsers/test_hermes_tool_parser.py b/tests/tool_parsers/test_hermes_tool_parser.py index 25869777dba5..13d7b598a711 100644 --- a/tests/tool_parsers/test_hermes_tool_parser.py +++ b/tests/tool_parsers/test_hermes_tool_parser.py @@ -412,3 +412,127 @@ def test_hermes_streaming_content_and_tool_call_in_single_chunk( assert tool_parts[0].function.name == "f" args_str = "".join(tc.function.arguments or "" for tc in tool_parts) assert json.loads(args_str) == {"x": 1} + + +# Reproduction from https://github.com/vllm-project/vllm/issues/45167 +END_TAG_IN_STRING_TEXT = ( + "\n" + '{"name": "edit_file", "arguments": ' + '{"content": "예시 태그: 포함 한국어 본문"}}\n' + "" +) + + +def test_hermes_parser_non_streaming_end_tag_inside_string( + qwen_tokenizer: TokenizerLike, + any_chat_request: ChatCompletionRequest, +) -> None: + """A literal in a string argument must not end the tool call.""" + parser = Hermes2ProToolParser(qwen_tokenizer) + tool_call = parser.extract_tool_calls( + model_output=END_TAG_IN_STRING_TEXT, + request=any_chat_request, + ) + + assert tool_call.tools_called + assert tool_call.content is None + assert len(tool_call.tool_calls) == 1 + assert tool_call.tool_calls[0].function.name == "edit_file" + args = json.loads(tool_call.tool_calls[0].function.arguments) + assert args["content"] == "예시 태그: 포함 한국어 본문" + + +def test_hermes_parser_non_streaming_end_tag_inside_first_of_two_calls( + qwen_tokenizer: TokenizerLike, + any_chat_request: ChatCompletionRequest, +) -> None: + """A quoted end tag must not swallow the tool call that follows it.""" + text = ( + '{"name": "first", ' + '"arguments": {"q": "a b"}}' + '{"name": "second", "arguments": {"q": "c"}}' + ) + parser = Hermes2ProToolParser(qwen_tokenizer) + tool_call = parser.extract_tool_calls( + model_output=text, + request=any_chat_request, + ) + + assert tool_call.tools_called + assert [tc.function.name for tc in tool_call.tool_calls] == ["first", "second"] + assert json.loads(tool_call.tool_calls[0].function.arguments) == { + "q": "a b" + } + assert json.loads(tool_call.tool_calls[1].function.arguments) == {"q": "c"} + + +def test_hermes_parser_non_streaming_escaped_quote_before_end_tag( + qwen_tokenizer: TokenizerLike, + any_chat_request: ChatCompletionRequest, +) -> None: + """Escaped quotes must not confuse the string tracking of the scanner.""" + text = ( + "" + '{"name": "say", "arguments": {"text": "say \\"hi\\" done"}}' + "" + ) + parser = Hermes2ProToolParser(qwen_tokenizer) + tool_call = parser.extract_tool_calls( + model_output=text, + request=any_chat_request, + ) + + assert tool_call.tools_called + assert tool_call.tool_calls[0].function.name == "say" + args = json.loads(tool_call.tool_calls[0].function.arguments) + assert args["text"] == 'say "hi" done' + + +def test_hermes_parser_streaming_end_tag_inside_string( + qwen_tokenizer: TokenizerLike, + any_chat_request: ChatCompletionRequest, +) -> None: + """Streaming a quoted end tag must yield the non-streaming tool call.""" + parser = Hermes2ProToolParser(qwen_tokenizer) + deltas = _simulate_streaming( + qwen_tokenizer, parser, any_chat_request, END_TAG_IN_STRING_TEXT + ) + + tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls] + assert {tc.index for tc in tool_calls} == {0} + assert tool_calls[0].function.name == "edit_file" + args = json.loads("".join(tc.function.arguments or "" for tc in tool_calls)) + assert "" in args["content"] + + # Compare against the text the deltas reconstruct: decoding one token at a + # time is lossy for multi-byte characters. + streamed_text = "".join( + qwen_tokenizer.decode([token]) + for token in qwen_tokenizer.encode(END_TAG_IN_STRING_TEXT) + ) + expected = Hermes2ProToolParser(qwen_tokenizer).extract_tool_calls( + model_output=streamed_text, + request=any_chat_request, + ) + assert args == json.loads(expected.tool_calls[0].function.arguments) + + +def test_hermes_parser_streaming_start_tag_inside_string( + qwen_tokenizer: TokenizerLike, + any_chat_request: ChatCompletionRequest, +) -> None: + """A quoted start tag must not open a second tool call while streaming.""" + text = ( + "" + '{"name": "edit_file", "arguments": ' + '{"content": "opens and closes "}}' + "" + ) + parser = Hermes2ProToolParser(qwen_tokenizer) + deltas = _simulate_streaming(qwen_tokenizer, parser, any_chat_request, text) + + tool_calls = [tc for d in deltas if d.tool_calls for tc in d.tool_calls] + assert {tc.index for tc in tool_calls} == {0} + assert tool_calls[0].function.name == "edit_file" + args = json.loads("".join(tc.function.arguments or "" for tc in tool_calls)) + assert args["content"] == "opens and closes " diff --git a/vllm/tool_parsers/hermes_tool_parser.py b/vllm/tool_parsers/hermes_tool_parser.py index c5b318cd19fc..dda736f3557f 100644 --- a/vllm/tool_parsers/hermes_tool_parser.py +++ b/vllm/tool_parsers/hermes_tool_parser.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import json -from collections.abc import Sequence +from collections.abc import Iterator, Sequence import regex as re @@ -25,7 +25,11 @@ Tool, ToolParser, ) -from vllm.tool_parsers.utils import is_complete_json, partial_tag_overlap +from vllm.tool_parsers.utils import ( + find_tag_outside_json_strings, + is_complete_json, + partial_tag_overlap, +) from vllm.utils.mistral import is_mistral_tokenizer logger = init_logger(__name__) @@ -35,9 +39,6 @@ class Hermes2ProToolParser(ToolParser): structural_tag_model = "hermes" tool_call_start_token: str = "" tool_call_end_token: str = "" - tool_call_regex = re.compile( - r"(.*?)|(.*)", re.DOTALL - ) scratch_pad_regex = re.compile(r"(.*?)", re.DOTALL) def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): @@ -67,6 +68,29 @@ def adjust_request( request.skip_special_tokens = False return request + def _iter_tool_call_bodies(self, text: str) -> Iterator[tuple[str, bool, bool]]: + """Yield the raw body of each tool call region in ``text``. + + The end token is located with JSON string awareness so that a literal + ```` inside a string argument does not end the region. A + region without an end token extends to the end of ``text``. + + Yields: + The body, whether the end token was found, and whether the body + ends inside an unterminated JSON string literal. + """ + pos = 0 + while (start := text.find(self.tool_call_start_token, pos)) != -1: + body_start = start + len(self.tool_call_start_token) + end, in_string = find_tag_outside_json_strings( + text, self.tool_call_end_token, body_start + ) + if end == -1: + yield text[body_start:], False, in_string + return + yield text[body_start:end], True, False + pos = end + len(self.tool_call_end_token) + def extract_tool_calls( self, model_output: str, @@ -80,17 +104,11 @@ def extract_tool_calls( else: try: - # there are two possible captures - between tags, or between a - # tag and end-of-string so the result of - # findall is an array of tuples where one is a function call and - # the other is None - function_call_tuples = self.tool_call_regex.findall(model_output) - # load the JSON, and then use it to build the Function and # Tool Call raw_function_calls = [ - json.loads(match[0] if match[0] else match[1]) - for match in function_call_tuples + json.loads(body) + for body, _, _ in self._iter_tool_call_bodies(model_output) ] tool_calls = [ ToolCall( @@ -141,28 +159,21 @@ def _extract_content(self, current_text: str) -> str | None: def _extract_tool_call_jsons(self, text: str) -> list[tuple[str, bool]]: """Extract (json_text, is_complete) for each region.""" results: list[tuple[str, bool]] = [] - pos = 0 - while True: - start = text.find(self.tool_call_start_token, pos) - if start == -1: - break - json_start = start + len(self.tool_call_start_token) - json_end = text.find(self.tool_call_end_token, json_start) - if json_end != -1: - results.append((text[json_start:json_end].strip(), True)) - pos = json_end + len(self.tool_call_end_token) - else: - raw = text[json_start:] - # Strip partial suffix if present. + for raw, terminated, in_string in self._iter_tool_call_bodies(text): + if terminated: + results.append((raw.strip(), True)) + continue + if not in_string: + # Strip partial suffix if present. Inside a + # string literal those characters are argument data. overlap = partial_tag_overlap(raw, self.tool_call_end_token) if overlap: raw = raw[:-overlap] - tc_json = raw.strip() - # Valid JSON without closing tag = complete body, - # tag tokens just haven't arrived yet. - is_complete = is_complete_json(tc_json) if tc_json else False - results.append((tc_json, is_complete)) - break + tc_json = raw.strip() + # Valid JSON without closing tag = complete body, + # tag tokens just haven't arrived yet. + is_complete = is_complete_json(tc_json) if tc_json else False + results.append((tc_json, is_complete)) return results @staticmethod diff --git a/vllm/tool_parsers/longcat_tool_parser.py b/vllm/tool_parsers/longcat_tool_parser.py index ccfa94167256..81d6e2d1a805 100644 --- a/vllm/tool_parsers/longcat_tool_parser.py +++ b/vllm/tool_parsers/longcat_tool_parser.py @@ -1,8 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import regex as re - from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import Tool from vllm.tool_parsers.hermes_tool_parser import Hermes2ProToolParser @@ -14,9 +12,3 @@ def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None): self.tool_call_start_token: str = "" self.tool_call_end_token: str = "" - - self.tool_call_regex = re.compile( - r"(.*?)" - r"|(.*)", - re.DOTALL, - ) diff --git a/vllm/tool_parsers/utils.py b/vllm/tool_parsers/utils.py index d39c83969f41..52653a1ca3c6 100644 --- a/vllm/tool_parsers/utils.py +++ b/vllm/tool_parsers/utils.py @@ -55,6 +55,46 @@ def partial_tag_overlap(text: str, tag: str) -> int: return 0 +def find_tag_outside_json_strings( + text: str, tag: str, start: int = 0 +) -> tuple[int, bool]: + """Find a tag that is not part of a JSON string literal. + + Scanning starts outside a string and tracks JSON string and escape state, + so a tag inside a string value is argument data rather than markup. + + Args: + text: The text to scan. + tag: The tag to look for, e.g. ``""``. + start: Index in ``text`` to start scanning from. + + Returns: + The index of the first such tag at or after ``start``, or -1 if there + is none, together with whether the scan reached the end of ``text`` + inside an unterminated string literal. + """ + pos = start + in_string = False + tag_pos = text.find(tag, start) + while pos < len(text): + quote = text.find('"', pos) + if in_string: + if quote == -1: + return -1, True + in_string = _is_escaped(text, quote) + pos = quote + 1 + continue + if 0 <= tag_pos < pos: + tag_pos = text.find(tag, pos) + if tag_pos != -1 and (quote == -1 or tag_pos < quote): + return tag_pos, False + if quote == -1: + return -1, False + in_string = True + pos = quote + 1 + return -1, in_string + + def find_common_prefix(s1: str, s2: str) -> str: """ Finds a common prefix that is shared between two strings, if there is one.