Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions tests/tool_parsers/test_structural_tag_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@
from vllm.tool_parsers.kimi_k2_tool_parser import KimiK2ToolParser
from vllm.tool_parsers.kimi_k3_tool_parser import KimiK3ToolParser
from vllm.tool_parsers.llama_tool_parser import Llama3JsonToolParser
from vllm.tool_parsers.longcat_tool_parser import LongcatFlashToolParser
from vllm.tool_parsers.mimo_tool_parser import MiMoToolParser
from vllm.tool_parsers.minimax_m2_tool_parser import MinimaxM2ToolParser
from vllm.tool_parsers.plamo3_engine_tool_parser import Plamo3EngineToolParser
Expand Down Expand Up @@ -850,6 +851,44 @@ class StepParser(DelegatingParser):
]


@pytest.mark.parametrize(
"tool_choice",
["required", {"type": "function", "function": {"name": "get_weather"}}],
)
def test_longcat_forced_tool_choice_round_trips_native_tags(
sample_tools: list[ChatCompletionToolsParam], tool_choice
):
"""Forced LongCat calls must use <longcat_tool_call>, not Hermes tags."""

class LongcatParser(DelegatingParser):
tool_parser_cls = LongcatFlashToolParser

request = ChatCompletionRequest(
messages=[],
model="m",
tools=[tool.model_dump(exclude_none=True) for tool in sample_tools],
tool_choice=tool_choice,
)
parser = LongcatParser(MagicMock(), tools=sample_tools)
request = parser.adjust_request(request)

grammar = Grammar.from_structural_tag(request.structured_outputs.structural_tag)
# Rendered exactly as the LongCat-Flash chat template renders tool calls.
output = (
'<longcat_tool_call>\n{"name": "get_weather", '
'"arguments": {"city": "Dallas"}}\n</longcat_tool_call>'
)
assert _is_grammar_accept_string(grammar, output)
assert not _is_grammar_accept_string(grammar, output.replace("longcat_", ""))

_, content, tool_calls = parser.parse(output, request, enable_auto_tools=True)

assert not content
assert [(call.name, json.loads(call.arguments)) for call in tool_calls] == [
("get_weather", {"city": "Dallas"})
]


def test_tool_parsers_without_structural_tag_support_required_and_named():
class NonStructuralTagToolParser(ToolParser):
pass
Expand Down
2 changes: 2 additions & 0 deletions vllm/tool_parsers/longcat_tool_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@


class LongcatFlashToolParser(Hermes2ProToolParser):
structural_tag_model = "longcat"

def __init__(self, tokenizer: TokenizerLike, tools: list[Tool] | None = None):
super().__init__(tokenizer, tools)

Expand Down
60 changes: 43 additions & 17 deletions vllm/tool_parsers/structural_tag_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@
}
)
VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset(
{"glm_4_7", "hermes", "hy_v4", "kimi_k3", "plamo3"}
{"glm_4_7", "hermes", "hy_v4", "kimi_k3", "longcat", "plamo3"}
)
SUPPORTED_STRUCTURAL_TAG_MODELS = (
XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS
Expand Down Expand Up @@ -346,15 +346,19 @@ def get_function_parameters(function) -> dict[str, Any] | bool:
return function.parameters if function.parameters is not None else True


def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
def _hermes_tool_tags(
tools: list[FunctionToolParam],
start_tag: str,
end_tag: str,
) -> list[TagFormat]:
arguments_field_prefix = '", "arguments": '
formats = [
# <tool_call>
# {"name": "t1", "arguments": {"q": "v"}}
# </tool_call>
('<tool_call>\n{"name": "', "}\n</tool_call>"),
(start_tag + '\n{"name": "', "}\n" + end_tag),
# <tool_call>{"name": "t1", "arguments": {"q": "v"}}</tool_call>
('<tool_call>{"name": "', "}</tool_call>"),
(start_tag + '{"name": "', "}" + end_tag),
]

return [
Expand All @@ -370,42 +374,64 @@ def _hermes_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
]


@register_vllm_structural_tag("hermes")
def get_hermes_structural_tag(
def _hermes_style_structural_tag(
tools: list[FunctionToolParam],
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
start_tag: str,
end_tag: str,
) -> StructuralTag:
del builtin_tools, reasoning, token_suffix

tool_call_trigger = "<tool_call>"

tags = _hermes_tool_tags(tools, start_tag, end_tag)
if tool_choice == "auto":
tags = _hermes_tool_tags(tools)
suffix_tag = (
TriggeredTagsFormat(triggers=[tool_call_trigger], tags=tags)
TriggeredTagsFormat(triggers=[start_tag], tags=tags)
if tags
else AnyTextFormat()
)
elif tool_choice == "forced":
suffix_tag = TagsWithSeparatorFormat(
tags=_hermes_tool_tags(tools),
tags=tags,
separator="",
at_least_one=True,
stop_after_first=True,
)
else:
suffix_tag = TagsWithSeparatorFormat(
tags=_hermes_tool_tags(tools),
tags=tags,
separator="",
at_least_one=True,
)

return StructuralTag(format=suffix_tag)


@register_vllm_structural_tag("hermes")
def get_hermes_structural_tag(
tools: list[FunctionToolParam],
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag:
del builtin_tools, reasoning, token_suffix
return _hermes_style_structural_tag(
tools, tool_choice, "<tool_call>", "</tool_call>"
)


@register_vllm_structural_tag("longcat")
def get_longcat_structural_tag(
tools: list[FunctionToolParam],
builtin_tools: list[BuiltinToolParam],
tool_choice: SimplifiedToolChoice,
reasoning: bool,
token_suffix: str = "",
) -> StructuralTag:
del builtin_tools, reasoning, token_suffix
return _hermes_style_structural_tag(
tools, tool_choice, "<longcat_tool_call>", "</longcat_tool_call>"
)


def _minimax_tool_tags(tools: list[FunctionToolParam]) -> list[TagFormat]:
return [
TagFormat(
Expand Down
Loading