Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions tests/tool_parsers/test_hunyuan_a13b_tool_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ def test_hunyuan_a13b_tool_parser_extract(
],
[make_tool_call("get_weather", {"city": "Boston"})],
),
pytest.param(
(
[
'<tool_calls>[{"name": "complex_tool",',
' "arguments": ',
Expand All @@ -156,9 +156,6 @@ def test_hunyuan_a13b_tool_parser_extract(
"complex_tool", {"level1": {"level2": {"level3": {"value": 123}}}}
)
],
marks=pytest.mark.xfail(
reason="stream parsing not support nested json yet."
),
),
],
)
Expand Down
162 changes: 103 additions & 59 deletions vllm/tool_parsers/hunyuan_a13b_tool_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def __init__(self, tokenizer: TokenizerLike):
self.current_tools_sent: list[bool] = []

# For backward compatibility with serving code
self.prev_tool_call_arr = []
self.prev_tool_call_arr: list[dict[str, Any]] = []

# Regex patterns for preprocessing
self.answer_tool_calls_pattern = re.compile(
Expand All @@ -57,7 +57,9 @@ def __init__(self, tokenizer: TokenizerLike):
r'"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*\{\s*\}'
)

# TODO: not support nested json object in fc arguments.
# Note: historically we used a regex for nested JSON. Streaming output
# is incremental, so regex brace matching can fail for deeply nested
# objects; we now use a brace-depth scanner instead.
self.tool_non_empty_arg_reg = re.compile(
r'"name"\s*:\s*"[^"]+"\s*,\s*"arguments"\s*:\s*(\{(?:[^{}]|(?:\{[^{}]*\}))*\})'
)
Expand Down Expand Up @@ -311,6 +313,65 @@ def _handle_tool_name_streaming(
return delta
return None

def _scan_braced_json_object(self, text: str, start_brace: int) -> str | None:
"""Extract a JSON object starting at `start_brace` (which must be '{').

In streaming mode the object may be incomplete; in that case we return
the partial text from `start_brace` to the end of `text`.
"""
assert start_brace < len(text) and text[start_brace] == "{"

depth = 0
in_string = False
escape = False

for i in range(start_brace, len(text)):
ch = text[i]
if in_string:
if escape:
escape = False
continue
if ch == "\\":
escape = True
continue
if ch == '"':
in_string = False
continue

if ch == '"':
in_string = True
continue

if ch == "{":
depth += 1
elif ch == "}":
depth -= 1
if depth == 0:
return text[start_brace : i + 1]

# Streaming/incomplete JSON: return whatever we have so far.
return text[start_brace:] if depth > 0 else None

def _extract_tool_arguments_text(
self, current_text: str, tool_index: int
) -> str | None:
"""Extract the `arguments` JSON object for a given tool index."""
tool_calls_start = current_text.find(self.bot_string)
search_start = tool_calls_start if tool_calls_start != -1 else 0

arguments_keys = list(
re.finditer(r'"arguments"\s*:', current_text[search_start:])
)
if tool_index >= len(arguments_keys):
return None

match = arguments_keys[tool_index]
after_colon = search_start + match.end()
brace_pos = current_text.find("{", after_colon)
if brace_pos == -1:
return None
return self._scan_braced_json_object(current_text, brace_pos)

def _handle_tool_args_streaming(
self, current_text: str, current_idx: int, tool_count: int
):
Expand Down Expand Up @@ -348,75 +409,58 @@ def _handle_tool_args_streaming(
]
return delta

args_matches = list(self.tool_non_empty_arg_reg.finditer(current_text))
if current_idx < len(args_matches):
args_text = args_matches[current_idx].group(1)
is_last_tool = current_idx == tool_count - 1
if not is_last_tool:
next_tool_pos = current_text.find(
"},{", args_matches[current_idx].start()
)
if next_tool_pos != -1:
args_end_pos = next_tool_pos + 1
args_text = (
current_text[
args_matches[current_idx].start() : args_end_pos
]
.split('"arguments":')[1]
.strip()
)
sent_args = self.streaming_state["sent_tools"][current_idx][
"sent_arguments"
]
if not self.streaming_state["sent_tools"][current_idx][
args_text = self._extract_tool_arguments_text(current_text, current_idx)

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

The new approach using _extract_tool_arguments_text and _scan_braced_json_object correctly handles nested JSON, which is a great improvement. However, this implementation introduces a performance concern. On each call (for each new token), these functions re-scan a large portion of the growing current_text. This leads to an overall quadratic time complexity (O(N^2)) for parsing the arguments stream, where N is the length of the arguments. In a performance-critical library like vLLM, this can be a significant issue for tool calls with large arguments.

Consider refactoring this to use a stateful streaming parser. The parser's state (current position, brace depth, etc.) could be stored in self.streaming_state and updated incrementally with each new token, avoiding the need to re-scan previously processed text. This would bring the complexity down to linear time (O(N)).

if args_text is None:
return None

sent_args = self.streaming_state["sent_tools"][current_idx][
"sent_arguments"
]
if not self.streaming_state["sent_tools"][current_idx][
"sent_arguments_prefix"
] and args_text.startswith("{"):
self.streaming_state["sent_tools"][current_idx][
"sent_arguments_prefix"
] and args_text.startswith("{"):
self.streaming_state["sent_tools"][current_idx][
"sent_arguments_prefix"
] = True
] = True
self.streaming_state["sent_tools"][current_idx]["sent_arguments"] = "{"
while len(self.streamed_args) <= current_idx:
self.streamed_args.append("")
self.streamed_args[current_idx] += "{"
delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=current_idx,
function=DeltaFunctionCall(arguments="{").model_dump(
exclude_none=True
),
)
]
)
return delta

if args_text.startswith(sent_args):
args_diff = args_text[len(sent_args) :]
if args_diff:
self.streaming_state["sent_tools"][current_idx][
"sent_arguments"
] = "{"
] = args_text
while len(self.streamed_args) <= current_idx:
self.streamed_args.append("")
self.streamed_args[current_idx] += "{"
self.streamed_args[current_idx] += args_diff
delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=current_idx,
function=DeltaFunctionCall(arguments="{").model_dump(
exclude_none=True
),
function=DeltaFunctionCall(
arguments=args_diff
).model_dump(exclude_none=True),
)
]
)
return delta

if args_text.startswith(sent_args):
args_diff = args_text[len(sent_args) :]
if args_diff:
self.streaming_state["sent_tools"][current_idx][
"sent_arguments"
] = args_text
while len(self.streamed_args) <= current_idx:
self.streamed_args.append("")
self.streamed_args[current_idx] += args_diff
delta = DeltaMessage(
tool_calls=[
DeltaToolCall(
index=current_idx,
function=DeltaFunctionCall(
arguments=args_diff
).model_dump(exclude_none=True),
)
]
)
return delta

if args_text.endswith("}") and args_text == sent_args:
if current_idx < tool_count - 1:
self.streaming_state["current_tool_index"] += 1
self.current_tool_id = self.streaming_state[
"current_tool_index"
]
if args_text.endswith("}") and args_text == sent_args:
if current_idx < tool_count - 1:
self.streaming_state["current_tool_index"] += 1
self.current_tool_id = self.streaming_state["current_tool_index"]
return None
Loading