diff --git a/tests/tool_parsers/test_gemma4_structural_tag.py b/tests/tool_parsers/test_gemma4_structural_tag.py new file mode 100644 index 000000000000..97682457d0bb --- /dev/null +++ b/tests/tool_parsers/test_gemma4_structural_tag.py @@ -0,0 +1,482 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +# Klang tests for the gemma4 structural-tag patch (vllm-project/vllm#50477, +# #53363). They follow the kimi_k3 tests in +# ``tests/tool_parsers/test_structural_tag_registry.py``. They fail on stock +# vLLM v0.32.0 and pass with the patch applied. The argument grammar needs the +# ``gemma`` JSON-schema style from mlc-ai/xgrammar#923, not yet in a release +# (see the README). + +from typing import Any + +import pytest +from xgrammar import Grammar, StructuralTag +from xgrammar.structural_tag import JSONSchemaFormat, TagFormat +from xgrammar.testing import _is_grammar_accept_string + +from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedFunction, + ChatCompletionNamedToolChoiceParam, + ChatCompletionRequest, + ChatCompletionToolsParam, +) +from vllm.parser.gemma4 import Gemma4Parser +from vllm.sampling_params import SamplingParams, StructuredOutputsParams +from vllm.tool_parsers.gemma4_engine_tool_parser import Gemma4EngineToolParser +from vllm.tool_parsers.structural_tag_registry import ( + VLLM_BUILTIN_STRUCTURAL_TAG_MODELS, + get_model_structural_tag, +) +from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar + + +class _DummyTokenizer: + def get_vocab(self): + return {} + + def encode(self, text, add_special_tokens=False): + return [ord(c) for c in text] + + +class _VocabTokenizer(_DummyTokenizer): + def get_vocab(self): + return { + "<|tool_call>": 1, + "": 2, + '<|"|>': 3, + "<|channel>": 4, + "": 5, + } + + def decode(self, token_ids, **kwargs): + by_id = {tid: text for text, tid in self.get_vocab().items()} + return "".join(by_id.get(tid, chr(tid)) for tid in token_ids) + + +def _tool( + name: str, + properties: dict[str, Any], + required: list[str] | None = None, + strict: bool | None = None, +) -> ChatCompletionToolsParam: + parameters: dict[str, Any] = {"type": "object", "properties": properties} + if required is not None: + parameters["required"] = required + function: dict[str, Any] = {"name": name, "parameters": parameters} + if strict is not None: + function["strict"] = strict + return ChatCompletionToolsParam(type="function", function=function) + + +def _gemma4_tools() -> list[ChatCompletionToolsParam]: + return [ + _tool( + "get_weather", + {"city": {"type": "string"}, "days": {"type": "integer"}}, + ["city"], + ), + _tool("run_command", {"command": {"type": "string"}}, ["command"]), + ] + + +def _gemma4_call(name: str, args: str) -> str: + return f"<|tool_call>call:{name}{{{args}}}" + + +def _gemma4_grammar(tool_choice, tools=None) -> Grammar: + tag = get_model_structural_tag( + model="gemma4", + tools=tools if tools is not None else _gemma4_tools(), + tool_choice=tool_choice, + reasoning=False, + ) + assert isinstance(tag, StructuralTag) + return Grammar.from_structural_tag(tag) + + +def _accepts(tool: ChatCompletionToolsParam, args: str) -> bool: + grammar = _gemma4_grammar("required", tools=[tool]) + return _is_grammar_accept_string(grammar, _gemma4_call(tool.function.name, args)) + + +def _call_tags(tools: list[ChatCompletionToolsParam]) -> list[TagFormat]: + tag = get_model_structural_tag( + model="gemma4", tools=tools, tool_choice="required", reasoning=False + ) + assert isinstance(tag, StructuralTag) + calls, _trailer = tag.format.elements + return calls.tags + + +def _request(tool_choice, tools=None) -> ChatCompletionRequest: + # Send the tools as raw dicts, the same shape as on the wire. + return ChatCompletionRequest( + model="gemma4", + messages=[{"role": "user", "content": "hi"}], + tools=[ + t.model_dump(exclude_none=True) + for t in (tools if tools is not None else _gemma4_tools()) + ], + tool_choice=tool_choice, + ) + + +def test_gemma4_registered_as_vllm_builtin(): + assert "gemma4" in VLLM_BUILTIN_STRUCTURAL_TAG_MODELS + assert Gemma4EngineToolParser.structural_tag_model == "gemma4" + assert not Gemma4EngineToolParser.supports_required_and_named + + +def test_gemma4_auto_without_strict_is_unconstrained(): + tag = get_model_structural_tag( + model="gemma4", + tools=_gemma4_tools(), + tool_choice="auto", + reasoning=False, + ) + assert tag is None + + +def test_gemma4_auto_with_strict_tool_allows_text_and_calls(): + tools = [_tool("get_weather", {"city": {"type": "string"}}, ["city"], strict=True)] + grammar = _gemma4_grammar("auto", tools=tools) + assert _is_grammar_accept_string(grammar, "Just answering.") + assert _is_grammar_accept_string( + grammar, "One moment. " + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>') + ) + + +@pytest.mark.parametrize( + "body", + [ + # single call, one string arg + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>'), + # two args, compact form + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>,days:3'), + # string value with spaces and metacharacters + _gemma4_call("run_command", 'command:<|"|>grep -E "a|b" x.py<|"|>'), + # two calls back-to-back + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>') + + _gemma4_call("run_command", 'command:<|"|>ls -la<|"|>'), + # trailing tool-response terminator + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>') + "<|tool_response>", + ], +) +def test_gemma4_required_accepts_valid_tool_calls(body: str): + assert _is_grammar_accept_string(_gemma4_grammar("required"), body) + + +@pytest.mark.parametrize( + "body", + [ + # required but no call + "no call here", + # free text before a required call + "Let me check. " + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>'), + # unknown tool name + _gemma4_call("get_temperature", 'city:<|"|>x<|"|>'), + # args out of the sorted property order + _gemma4_call("get_weather", 'days:3,city:<|"|>Paris<|"|>'), + # integer arg with a non-numeric value + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>,days:abc'), + # undeclared argument key + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>,zzz:<|"|>x<|"|>'), + # empty args while the schema has required properties + _gemma4_call("get_weather", ""), + # unterminated call + '<|tool_call>call:get_weather{city:<|"|>Paris', + # trailing junk after the call + _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>') + "and more text", + ], +) +def test_gemma4_required_rejects_invalid(body: str): + assert not _is_grammar_accept_string(_gemma4_grammar("required"), body) + + +def test_gemma4_schema_without_required_accepts_empty_call(): + tools = [_tool("get_weather", {"city": {"type": "string"}})] + grammar = _gemma4_grammar("required", tools=tools) + assert _is_grammar_accept_string(grammar, _gemma4_call("get_weather", "")) + + +def test_gemma4_tool_without_properties_accepts_only_empty_args(): + tool = _tool("ping", {}) + assert _accepts(tool, "") + assert not _accepts(tool, 'why:<|"|>because<|"|>') + + +@pytest.mark.parametrize( + "function", + [ + # a parameters dict that declares no object schema + {"name": "ping", "parameters": {}}, + # strict=False, whatever the schema says + { + "name": "ping", + "strict": False, + "parameters": { + "type": "object", + "properties": {"why": {"type": "string"}}, + }, + }, + # no parameters at all + {"name": "ping"}, + ], +) +def test_gemma4_unconstrainable_parameters_still_force_the_syntax(function: dict): + grammar = _gemma4_grammar( + "required", tools=[ChatCompletionToolsParam(type="function", function=function)] + ) + assert _is_grammar_accept_string(grammar, _gemma4_call("ping", "")) + assert _is_grammar_accept_string(grammar, _gemma4_call("ping", "why:1,how:two")) + # the braces come from the tag here, so a bare value is not a call: without + # them the parser reads ``call:ping5`` as a tool named ping5 with no args + assert not _is_grammar_accept_string(grammar, "<|tool_call>call:ping5") + + +@pytest.mark.parametrize( + "parameters", + [ + pytest.param( + {"type": "object", "patternProperties": {"^x_": {"type": "string"}}}, + id="patternProperties", + ), + pytest.param( + { + "type": "object", + "properties": {"room number": {"type": "string"}}, + "required": ["room number"], + }, + id="non_identifier_key", + ), + ], +) +def test_gemma4_schema_with_bare_key_patterns_stays_constrained( + parameters: dict, monkeypatch +): + monkeypatch.setenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "1") + tool = ChatCompletionToolsParam( + type="function", function={"name": "book_room", "parameters": parameters} + ) + (call_tag,) = _call_tags([tool]) + assert call_tag.begin == "<|tool_call>call:book_room" + assert isinstance(call_tag.content, JSONSchemaFormat) + + parser = Gemma4Parser(_DummyTokenizer()) + request = parser.adjust_request(_request("required", [tool])) + # Raises VLLMValidationError -> HTTP 400 if the tag does not compile. + validate_xgrammar_grammar( + SamplingParams( + structured_outputs=StructuredOutputsParams( + structural_tag=request.structured_outputs.structural_tag + ) + ) + ) + + +def test_gemma4_compilable_schema_keeps_the_constrained_path(): + weather, _command = _call_tags(_gemma4_tools()) + assert weather.begin == "<|tool_call>call:get_weather" + assert isinstance(weather.content, JSONSchemaFormat) + + +def test_gemma4_open_parameter_bag_constrains_only_the_declared_key(): + # Models do_action: only action_id is typed, the rest is an open bag. + tool = ChatCompletionToolsParam( + type="function", + function={ + "name": "do_action", + "parameters": { + "type": "object", + "properties": {"action_id": {"type": "string"}}, + "required": ["action_id"], + "additionalProperties": True, + }, + }, + ) + assert _accepts(tool, 'action_id:<|"|>bake<|"|>,duration:5') + # Keys stay bare. The delimiter wraps values only. + assert not _accepts(tool, 'action_id:<|"|>bake<|"|>,<|"|>duration<|"|>:5') + + +def test_gemma4_properties_follow_the_sorted_order(): + # The chat template dictsorts the parameters, so the model emits the keys + # case-insensitively sorted rather than in declaration order. + tool = _tool( + "plan_step", + {"zeta": {"type": "string"}, "alpha": {"type": "string"}}, + ["zeta", "alpha"], + ) + assert _accepts(tool, 'alpha:<|"|>a<|"|>,zeta:<|"|>z<|"|>') + assert not _accepts(tool, 'zeta:<|"|>z<|"|>,alpha:<|"|>a<|"|>') + + +def test_gemma4_seed_tool_declared_out_of_sorted_order(): + # submit_interaction_summary is one of the four SEED tools whose + # declaration order differs from the sorted one, so it is where a flip of + # the registry's any_order would show. + tool = _tool( + "submit_interaction_summary", + { + "summary": {"type": "string"}, + "perception_updates": {"type": "array", "items": {"type": "string"}}, + }, + ["summary", "perception_updates"], + ) + assert _accepts(tool, 'perception_updates:[<|"|>p<|"|>],summary:<|"|>s<|"|>') + assert not _accepts(tool, 'summary:<|"|>s<|"|>,perception_updates:[<|"|>p<|"|>]') + + +def test_gemma4_nested_properties_follow_the_sorted_order(): + tool = _tool( + "plan_route", + { + "route": { + "type": "object", + "properties": { + "Zone": {"type": "string"}, + "alpha": {"type": "integer"}, + }, + "required": ["Zone", "alpha"], + } + }, + ["route"], + ) + assert _accepts(tool, 'route:{alpha:1,Zone:<|"|>z<|"|>}') + assert not _accepts(tool, 'route:{Zone:<|"|>z<|"|>,alpha:1}') + + +def test_gemma4_string_value_rejects_the_call_markers(): + # A marker inside a string value would desync the parser's FSM, which has + # no string awareness. + tool = _tool("run_command", {"command": {"type": "string"}}, ["command"]) + assert not _accepts(tool, 'command:<|"|>lsrm -rf /<|"|>') + assert not _accepts(tool, 'command:<|"|>ls<|tool_call>rm -rf /<|"|>') + + +def test_gemma4_object_property_is_constrained(): + tool = _tool( + "update_state", + { + "payload": { + "type": "object", + "properties": {"inner": {"type": "string"}}, + "required": ["inner"], + } + }, + ["payload"], + ) + assert _accepts(tool, 'payload:{inner:<|"|>v<|"|>}') + assert not _accepts(tool, 'payload:{other:<|"|>v<|"|>}') + + +def test_gemma4_array_property_respects_item_type_and_bounds(): + tool = _tool( + "render_speech", + { + "lines": { + "type": "array", + "items": {"type": "string"}, + "minItems": 1, + "maxItems": 2, + } + }, + ["lines"], + ) + assert _accepts(tool, 'lines:[<|"|>hello<|"|>]') + assert _accepts(tool, 'lines:[<|"|>hello<|"|>,<|"|>there<|"|>]') + assert not _accepts(tool, "lines:[]") + assert not _accepts(tool, 'lines:[<|"|>a<|"|>,<|"|>b<|"|>,<|"|>c<|"|>]') + assert not _accepts(tool, 'lines:{lines:[<|"|>hello<|"|>]}') + + +def test_gemma4_array_of_strings_rejects_objects(): + tool = _tool( + "pick_facts", + {"fact_ids": {"type": "array", "items": {"type": "string"}}}, + ["fact_ids"], + ) + assert _accepts(tool, 'fact_ids:[<|"|>26<|"|>]') + assert not _accepts(tool, 'fact_ids:[{fact_id:<|"|>26<|"|>}]') + + +def test_gemma4_number_and_optional_enum_are_typed(): + tool = _tool( + "score_reply", + { + "charge": {"type": "number"}, + "response_type": {"type": "string", "enum": ["chat", "action"]}, + }, + ["charge"], + ) + assert _accepts(tool, "charge:1.5") + assert _accepts(tool, 'charge:1.5,response_type:<|"|>chat<|"|>') + assert not _accepts(tool, 'charge:<|"|>1.5<|"|>') + assert not _accepts(tool, 'charge:1.5,response_type:<|"|>other<|"|>') + # the required charge is missing + assert not _accepts(tool, 'response_type:<|"|>chat<|"|>') + + +def test_gemma4_forced_tool_choice_builds_single_mandatory_call(): + grammar = _gemma4_grammar( + ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="get_weather") + ) + ) + ok = _gemma4_call("get_weather", 'city:<|"|>Paris<|"|>') + assert _is_grammar_accept_string(grammar, ok) + assert not _is_grammar_accept_string( + grammar, _gemma4_call("run_command", 'command:<|"|>ls<|"|>') + ) + # forced stops after the first call + assert not _is_grammar_accept_string(grammar, ok + ok) + assert not _is_grammar_accept_string(grammar, "just text") + + +def test_unified_parser_attaches_structural_tag_for_required(monkeypatch): + monkeypatch.setenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "1") + parser = Gemma4Parser(_DummyTokenizer()) + request = parser.adjust_request(_request("required")) + assert request.structured_outputs is not None + assert request.structured_outputs.structural_tag is not None + assert '"style": "gemma"' in request.structured_outputs.structural_tag + assert request.skip_special_tokens is False + + +def test_unified_parser_attaches_structural_tag_for_named(monkeypatch): + monkeypatch.setenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "1") + parser = Gemma4Parser(_DummyTokenizer()) + request = parser.adjust_request( + _request({"type": "function", "function": {"name": "get_weather"}}) + ) + assert request.structured_outputs is not None + assert request.structured_outputs.structural_tag is not None + + +def test_unified_parser_leaves_none_tool_choice_unconstrained(): + parser = Gemma4Parser(_DummyTokenizer()) + request = parser.adjust_request(_request("none")) + assert request.structured_outputs is None + + +def test_tool_parser_builds_structural_tag_for_delegating_path(monkeypatch): + monkeypatch.setenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "1") + parser = Gemma4EngineToolParser(_DummyTokenizer()) + tag = parser.get_structural_tag(_request("required")) + assert isinstance(tag, StructuralTag) + + +def test_parse_extracts_text_spelled_markers_with_token_ids(monkeypatch): + monkeypatch.setenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "1") + parser = Gemma4Parser(_VocabTokenizer()) + request = _request("required") + parser.adjust_request(request) + out = '<|tool_call>call:get_weather{city:<|"|>Paris<|"|>}' + ids = [ord(c) for c in out] + reasoning, content, tool_calls = parser.parse( + out, request, enable_auto_tools=True, model_output_token_ids=ids + ) + assert tool_calls is not None and len(tool_calls) == 1 + assert tool_calls[0].name == "get_weather" + assert content is None diff --git a/vllm/parser/gemma4.py b/vllm/parser/gemma4.py index 2a6246041d5f..96e443849974 100644 --- a/vllm/parser/gemma4.py +++ b/vllm/parser/gemma4.py @@ -312,11 +312,15 @@ def gemma4_config(thinking: bool = False) -> ParserEngineConfig: "OPEN_BRACE": "{", "OPEN_PAREN": "(", }, + # Klang patch: TOOL_START and TOOL_END are text terminals only. With + # a token id registered, the engine demotes text matches of these + # markers to content when the stream has token ids (see ``strict`` in + # ``streaming_parser_engine.py``). The structural tag forces the + # markers as text, and xgrammar lets the model spell them with + # regular tokens, so id-only matching drops the forced calls. token_id_terminals={ "THINK_START": CHANNEL_START, "THINK_END": CHANNEL_END, - "TOOL_START": TOOL_CALL_START, - "TOOL_END": TOOL_CALL_END, }, transitions={ # -- Reasoning transitions -- @@ -402,6 +406,68 @@ def gemma4_config(thinking: bool = False) -> ParserEngineConfig: ) +# Klang patch (not upstream): apply the gemma4 structural tag on the unified +# engine path. When the reasoning parser and the tool parser are both +# ``gemma4``, ``ParserManager.get_parser`` returns this engine directly, so +# ``DelegatingParser._apply_structural_tag`` never runs and required/named +# tool choice is not enforced (vllm-project/vllm#50477, #53363). +# The hook also fires when gemma4 is only the reasoning parser, because the +# reasoning adapter delegates ``adjust_request`` to this engine. That +# attaches the gemma4 grammar for a different tool parser, so do not run +# this image with ``--reasoning-parser gemma4`` and another tool parser. +def _apply_gemma4_structural_tag( + request: ChatCompletionRequest | ResponsesRequest, +) -> ChatCompletionRequest | ResponsesRequest: + import vllm.envs as envs + + if not envs.VLLM_ENFORCE_STRICT_TOOL_CALLING or not getattr( + request, "tools", None + ): + return request + + from openai.types.responses import ToolChoiceFunction + + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionNamedToolChoiceParam, + ) + + tool_choice = getattr(request, "tool_choice", None) + if not ( + tool_choice in ("auto", "required") + or isinstance( + tool_choice, (ChatCompletionNamedToolChoiceParam, ToolChoiceFunction) + ) + ): + return request + + structured_outputs = getattr(request, "structured_outputs", None) + if structured_outputs is not None and structured_outputs.structural_tag is not None: + return request + + from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag + + structure_tag = get_model_structural_tag( + model="gemma4", + tools=request.tools, + tool_choice=tool_choice, + reasoning=False, + ) + if structure_tag is None: + return request + + from vllm.entrypoints.openai.responses.protocol import ResponsesRequest + from vllm.sampling_params import StructuredOutputsParams + + request.structured_outputs = StructuredOutputsParams( + structural_tag=json.dumps(structure_tag.model_dump()) + ) + if isinstance(request, ResponsesRequest): + request.text = None + else: + request.response_format = None + return request + + _GEMMA4_THOUGHT_PREFIX = "thought\n" _GEMMA4_THOUGHT_TOKEN = "thought" @@ -443,6 +509,12 @@ def _reset(self, initial_state=None) -> None: self._prefix_stripped = False self._is_first_feed = True + def adjust_request( + self, request: ChatCompletionRequest | ResponsesRequest + ) -> ChatCompletionRequest | ResponsesRequest: + request = _apply_gemma4_structural_tag(request) + return super().adjust_request(request) + def _preprocess_feed( self, delta_text: str, diff --git a/vllm/tool_parsers/gemma4_engine_tool_parser.py b/vllm/tool_parsers/gemma4_engine_tool_parser.py index fcb578b129cb..a27dfd55973f 100644 --- a/vllm/tool_parsers/gemma4_engine_tool_parser.py +++ b/vllm/tool_parsers/gemma4_engine_tool_parser.py @@ -13,6 +13,10 @@ class Gemma4EngineToolParser(Gemma4ParserToolAdapter): # type: ignore[valid-type, misc] supports_required_and_named = False + # Klang patch (not upstream): let ``DelegatingParser._apply_structural_tag`` + # constrain decoding to the native format for required/named tool choice + # (vllm-project/vllm#50477, #53363). + structural_tag_model = "gemma4" def adjust_request( self, request: ChatCompletionRequest | ResponsesRequest diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 7e5902b8c2f6..3198db5bf316 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import functools import json from collections.abc import Callable, Sequence from typing import Any, Literal, TypeAlias @@ -12,7 +13,7 @@ from openai.types.responses.tool_choice_allowed import ToolChoiceAllowed from openai.types.responses.tool_choice_function import ToolChoiceFunction from pydantic import BaseModel -from xgrammar import StructuralTag, normalize_tool_choice +from xgrammar import Grammar, StructuralTag, normalize_tool_choice from xgrammar import get_model_structural_tag as get_xgrammar_model_structural_tag from xgrammar.openai_tool_call_schema import ( BuiltinToolParam, @@ -38,6 +39,7 @@ ChatCompletionNamedToolChoiceParam, ChatCompletionToolsParam, ) +from vllm.logger import init_logger from vllm.tool_parsers.tool_strict_level import ToolStrictLevel from vllm.tool_parsers.utils import ( custom_tool_parameters, @@ -45,6 +47,8 @@ require_function_tools, ) +logger = init_logger(__name__) + ToolChoice: TypeAlias = ( Literal["none", "auto", "required"] | ChatCompletionNamedToolChoiceParam @@ -82,8 +86,9 @@ "deepseek_v4_1", } ) +# "gemma4" is a Klang patch, not upstream. VLLM_BUILTIN_STRUCTURAL_TAG_MODELS = frozenset( - {"glm_4_7", "hermes", "hy_v4", "kimi_k3", "longcat", "plamo3"} + {"gemma4", "glm_4_7", "hermes", "hy_v4", "kimi_k3", "longcat", "plamo3"} ) SUPPORTED_STRUCTURAL_TAG_MODELS = ( XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS | VLLM_BUILTIN_STRUCTURAL_TAG_MODELS @@ -212,6 +217,8 @@ def get_model_structural_tag( tools = get_structural_tag_tools(model, tools, tool_choice) if not tools: return None + if model == "gemma4": + tools = _gemma4_strict_by_default(tools, tool_choice) tools = resolve_tool_strictness(tools, tool_choice, strict_level) if tools is None: return None @@ -1164,3 +1171,186 @@ def get_glm_4_7_structural_tag( ) return StructuralTag(format=SequenceFormat(elements=[prefix_tag, suffix_tag])) return StructuralTag(format=suffix_tag) + + +# --------------------------------------------------------------------------- +# Gemma4 (native ``<|tool_call>call:...`` format) +# --------------------------------------------------------------------------- +# Gemma4 assistant tool calls (see ``vllm/parser/gemma4.py``): +# <|tool_call>call:NAME{key:<|"|>string val<|"|>,count:42,flag:true} +# Multiple calls are emitted back-to-back with no separator. The argument block +# is JSON-shaped but not JSON: keys are bare and string values are wrapped in +# the ``<|"|>`` delimiter token and emitted raw, without escapes. xgrammar +# expresses that shape as the ``gemma`` JSON-schema style, so the tool's +# parameters schema constrains the whole block, its braces included. +_GEMMA4_TOOL_CALL_START = "<|tool_call>" +_GEMMA4_TOOL_CALL_END = "" +# The proper post-call terminator. Some checkpoints emit EOS directly +# instead, so the trailer is optional. +_GEMMA4_TOOL_RESPONSE = "<|tool_response>" +# False is the only setting that enforces required keys and uniqueness. It also +# pins the property order to the schema's order, which +# ``_gemma4_dictsort_schema`` sets to the chat template's ``dictsort`` order, +# the order the model emits. +_GEMMA4_ANY_ORDER = False +_GEMMA4_ARGUMENT_EXCLUDES = [_GEMMA4_TOOL_CALL_START, _GEMMA4_TOOL_CALL_END] + + +def _gemma4_dictsort(mapping: dict[str, Any], value: Callable[[Any], Any]) -> dict: + return { + key: value(item) + for key, item in sorted(mapping.items(), key=lambda kv: kv[0].lower()) + } + + +def _gemma4_dictsort_literal(literal: Any) -> Any: + if isinstance(literal, dict): + return _gemma4_dictsort(literal, _gemma4_dictsort_literal) + if isinstance(literal, list): + return [_gemma4_dictsort_literal(item) for item in literal] + return literal + + +def _gemma4_dictsort_schema(schema: Any) -> Any: + if isinstance(schema, list): + return [_gemma4_dictsort_schema(item) for item in schema] + if not isinstance(schema, dict): + return schema + sorted_schema = {} + for key, item in schema.items(): + if key == "properties" and isinstance(item, dict): + sorted_schema[key] = _gemma4_dictsort(item, _gemma4_dictsort_schema) + elif key in ("const", "enum", "default", "examples"): + sorted_schema[key] = _gemma4_dictsort_literal(item) + else: + sorted_schema[key] = _gemma4_dictsort_schema(item) + return sorted_schema + + +def _gemma4_syntax_only_tag(name: str) -> TagFormat: + # No schema is applied here, so without these literal braces the model + # could emit a bare call name that the parser would read as a call with + # no arguments. + return TagFormat( + begin=f"{_GEMMA4_TOOL_CALL_START}call:{name}{{", + content=AnyTextFormat( + excludes=[_GEMMA4_TOOL_CALL_START, _GEMMA4_TOOL_CALL_END] + ), + end="}" + _GEMMA4_TOOL_CALL_END, + ) + + +def _gemma4_strict_by_default( + tools: Sequence[ChatCompletionToolsParam | ResponsesTool], + tool_choice: ToolChoice, +) -> Sequence[ChatCompletionToolsParam | ResponsesTool]: + if tool_choice == "auto" and not _any_tool_strict(tools): + return tools + return [ + _with_tool_strict(tool, True) if _tool_strict_unset(tool) else tool + for tool in tools + ] + + +def _tool_strict_unset(tool: ChatCompletionToolsParam | ResponsesTool) -> bool: + if isinstance(tool, FunctionTool): + return tool.strict is None + if isinstance(tool, ChatCompletionToolsParam): + return tool.function.strict is None + return False + + +@functools.lru_cache(maxsize=256) +def _gemma4_schema_compiles(name: str, schema_json: str) -> bool: + # The style raises on a shape it cannot express: patternProperties, + # propertyNames, a property key that is not an identifier, an invalid + # pattern. That reaches the caller as an HTTP 400 on every request + # carrying the tool, so probe the schema here and degrade to the + # syntax-only tag instead. + try: + Grammar.from_structural_tag( + StructuralTag( + format=JSONSchemaFormat( + json_schema=json.loads(schema_json), + style="gemma", + any_order=_GEMMA4_ANY_ORDER, + excludes=_GEMMA4_ARGUMENT_EXCLUDES, + ) + ) + ) + except Exception as exc: + logger.warning( + "Gemma4 cannot constrain the parameters of tool %s (%s); " + "its arguments stay unconstrained.", + name, + " ".join(str(exc).split()), + ) + return False + return True + + +def _gemma4_call_tag(tool: FunctionToolParam) -> TagFormat: + function = tool.function + params = get_function_parameters(function) + if not isinstance(params, dict) or params.get("type") != "object": + # When strict is False, there are no parameters, or the schema does + # not declare an object, constrain the syntax only. + return _gemma4_syntax_only_tag(function.name) + params = _gemma4_dictsort_schema(params) + if not _gemma4_schema_compiles(function.name, json.dumps(params, sort_keys=True)): + return _gemma4_syntax_only_tag(function.name) + return TagFormat( + begin=f"{_GEMMA4_TOOL_CALL_START}call:{function.name}", + content=JSONSchemaFormat( + json_schema=params, + style="gemma", + any_order=_GEMMA4_ANY_ORDER, + excludes=_GEMMA4_ARGUMENT_EXCLUDES, + ), + end=_GEMMA4_TOOL_CALL_END, + ) + + +@register_vllm_structural_tag("gemma4") +def get_gemma4_structural_tag( + tools: list[FunctionToolParam], + builtin_tools: list[BuiltinToolParam], + tool_choice: SimplifiedToolChoice, + reasoning: bool, + token_suffix: str = "", +) -> StructuralTag: + del builtin_tools, reasoning, token_suffix + + call_tags = [_gemma4_call_tag(tool) for tool in tools] + + if tool_choice == "auto": + suffix_tag: Any = ( + TriggeredTagsFormat(triggers=[_GEMMA4_TOOL_CALL_START], tags=call_tags) + if call_tags + else AnyTextFormat() + ) + return StructuralTag(format=suffix_tag) + + if not call_tags: + return StructuralTag(format=AnyTextFormat()) + + calls = TagsWithSeparatorFormat( + tags=call_tags, + separator="", + at_least_one=True, + stop_after_first=(tool_choice == "forced"), + ) + # The first constrained token must already open a call. The engine + # applies the grammar only after the reasoning block ends (see + # ``vllm/v1/structured_output/__init__.py``), so a thought prefix needs + # no allowance here. A free-text preamble must not be allowed: with EOS + # masked, the model can then write text until ``max_tokens`` and never + # start a call. + return StructuralTag( + format=SequenceFormat( + elements=[ + calls, + OptionalFormat(content=ConstStringFormat(value=_GEMMA4_TOOL_RESPONSE)), + ] + ) + ) diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index 0199c5f75d6f..3a85bd96b0ef 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -107,7 +107,7 @@ def compile_grammar( # Upd. this bug was fixed in xgrammar main branch on Oct 8, 2026 # and will be available in next release. # So, remove this comment once xgrammar updates to v0.2.9 - separators = (",", ":") if self.disable_any_whitespace else None + separators = (", ", ": ") if self.disable_any_whitespace else None if request_type == StructuredOutputOptions.JSON: ctx = self.compiler.compile_json_schema( grammar_spec,