Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions docs/features/tool_calling.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ Whether vLLM enforces the tool parameter schema during generation depends on the
| --- | --- | --- |
| Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. |
| `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. |
| `"auto"` | Only when `strict: true` is set on at least one tool | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`. Without it, the model generates freely and tool calls are extracted from raw text. |
| `"auto"` | When `strict: true` is set on at least one tool, or the server override is enabled | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`, or when `VLLM_ENFORCE_STRICT_TOOL_CALLING=true`. Otherwise, the model generates freely and tool calls are extracted from raw text. |
| `"none"` | N/A | No tool calls are produced. |

### Strict Mode
Expand All @@ -128,9 +128,10 @@ For best compatibility with strict schema enforcement, define tool parameter sch
* Mark all fields in `properties` as required.
* Represent optional fields by allowing `null`, for example `{"type": ["string", "null"]}`.

vLLM also provides a global toggle via the `VLLM_ENFORCE_STRICT_TOOL_CALLING` environment variable (defaults to `true`). When set to `false`, vLLM does not attach structural tags for tool calling regardless of the per-tool `strict` field. This environment variable only affects structural-tag based tool calling; it does not change schema-derived structured outputs used by named function calling or `tool_choice="required"`.
vLLM also provides a global override via the `VLLM_ENFORCE_STRICT_TOOL_CALLING` environment variable. When unset (the default), the request's per-tool `strict` fields control automatic tool calling as described above. Set it to `true` to attach structural tags and enforce every tool's parameter schema even when clients omit `strict`, or to `false` to disable structural tags regardless of the per-tool field. This override only affects structural-tag based tool calling; it does not change schema-derived structured outputs used by named function calling or `tool_choice="required"`.

```bash
VLLM_ENFORCE_STRICT_TOOL_CALLING=true vllm serve ...
VLLM_ENFORCE_STRICT_TOOL_CALLING=false vllm serve ...
```

Expand All @@ -152,7 +153,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso
If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template!

!!! note
With `tool_choice="auto"`, schema-level constraint requires both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`. When these conditions are met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.
With `tool_choice="auto"`, schema-level constraint requires either at least one tool with `strict: true` or the server override `VLLM_ENFORCE_STRICT_TOOL_CALLING=true`. When either condition is met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema.

### Hermes Models (`hermes`)

Expand Down
2 changes: 1 addition & 1 deletion tests/parser/test_parse.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

@pytest.fixture(autouse=True)
def enable_hermes_required_named_parsing(monkeypatch):
# With VLLM_ENFORCE_STRICT_TOOL_CALLING (default on), Hermes sets
# Unless strict tool calling is explicitly disabled, Hermes sets
# supports_required_and_named=False and uses structural-tag guided tool
# calling. Force it True to test the non-guided JSON required/named parse path.
monkeypatch.setattr(Hermes2ProToolParser, "supports_required_and_named", True)
Expand Down
2 changes: 1 addition & 1 deletion tests/parser/test_streaming.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@

@pytest.fixture(autouse=True)
def enable_hermes_required_named_parsing(monkeypatch):
# With VLLM_ENFORCE_STRICT_TOOL_CALLING (default on), Hermes sets
# Unless strict tool calling is explicitly disabled, Hermes sets
# supports_required_and_named=False and uses structural-tag guided tool
# calling. Force it True to test the non-guided JSON required/named parse path.
monkeypatch.setattr(Hermes2ProToolParser, "supports_required_and_named", True)
Expand Down
164 changes: 164 additions & 0 deletions tests/tool_parsers/test_structural_tag_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
from unittest.mock import MagicMock

import pytest
from openai.types.responses import FunctionTool
from xgrammar import Grammar, StructuralTag
from xgrammar.structural_tag import TagsWithSeparatorFormat, TriggeredTagsFormat
from xgrammar.testing import _is_grammar_accept_string

from vllm.entrypoints.openai.chat_completion.protocol import (
Expand Down Expand Up @@ -679,3 +681,165 @@ def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call():
response_only = _k3_response("no call here")
assert _is_grammar_accept_string(grammar, ok)
assert not _is_grammar_accept_string(grammar, response_only)


class TestEnforceStrictToolCalling:
"""Server override behavior for structural-tag based tool calling."""

@pytest.fixture
def dumped_tools(self, monkeypatch: pytest.MonkeyPatch):
"""Capture the tool dicts handed to xgrammar."""
captured: list[list[dict]] = []

def fake_get_xgrammar_model_structural_tag(*, tools: list[dict], **kwargs):
captured.append(tools)
return MagicMock(spec=StructuralTag)

monkeypatch.setattr(
"vllm.tool_parsers.structural_tag_registry."
"get_xgrammar_model_structural_tag",
fake_get_xgrammar_model_structural_tag,
)
return captured

@pytest.mark.parametrize(
("value", "request_strict", "expect_tag"),
[
(None, False, False),
(None, True, True),
("true", False, True),
("false", True, False),
],
)
def test_auto_follows_explicit_server_override(
self,
monkeypatch: pytest.MonkeyPatch,
sample_tools: list[ChatCompletionToolsParam],
sample_tools_strict: list[ChatCompletionToolsParam],
value: str | None,
request_strict: bool,
expect_tag: bool,
):
if value is None:
monkeypatch.delenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", raising=False)
else:
monkeypatch.setenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", value)
tools = sample_tools_strict if request_strict else sample_tools
request = ChatCompletionRequest(
messages=[], model="test-model", tools=tools, tool_choice="auto"
)
parser = Qwen3EngineToolParser(MagicMock(), tools=tools)

tag = parser.get_structural_tag(request)

assert (tag is not None) is expect_tag

def test_force_strict_overrides_tools_without_mutating_request(
self,
dumped_tools: list[list[dict]],
sample_tools: list[ChatCompletionToolsParam],
):
explicitly_relaxed = ChatCompletionToolsParam(
type="function",
function={
"name": "get_time",
"strict": False,
"parameters": {"type": "object", "properties": {}},
},
)
responses_tool = FunctionTool(
type="function",
name="get_location",
strict=False,
parameters={"type": "object", "properties": {}},
)
tools = [*sample_tools, explicitly_relaxed, responses_tool]

get_model_structural_tag(
model="llama",
tools=tools,
tool_choice="auto",
reasoning=False,
force_strict=True,
)

assert dumped_tools[0][0]["function"]["strict"] is True
assert dumped_tools[0][1]["function"]["strict"] is True
assert dumped_tools[0][2]["function"]["strict"] is True
assert sample_tools[0].function.strict is None
assert explicitly_relaxed.function.strict is False
assert responses_tool.strict is False

@pytest.mark.parametrize(
("value", "expected_support"),
[(None, False), ("true", False), ("false", True)],
)
def test_parser_routing_follows_explicit_server_override(
self,
monkeypatch: pytest.MonkeyPatch,
value: str | None,
expected_support: bool,
):
if value is None:
monkeypatch.delenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", raising=False)
else:
monkeypatch.setenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", value)

class TestParser(ToolParser):
structural_tag_model = "llama"

assert TestParser.supports_required_and_named is expected_support

def test_force_strict_keeps_text_response_allowed_under_auto(
self,
sample_tools: list[ChatCompletionToolsParam],
):
auto_tag = get_model_structural_tag(
model="hermes",
tools=sample_tools,
tool_choice="auto",
reasoning=False,
force_strict=True,
)
required_tag = get_model_structural_tag(
model="hermes",
tools=sample_tools,
tool_choice="required",
reasoning=False,
force_strict=True,
)

assert auto_tag is not None
assert required_tag is not None
assert isinstance(auto_tag.format, TriggeredTagsFormat)
assert isinstance(required_tag.format, TagsWithSeparatorFormat)

def test_force_strict_supports_parallel_tool_calls(
self,
sample_tools: list[ChatCompletionToolsParam],
):
tools = sample_tools + [
ChatCompletionToolsParam(
type="function",
function={
"name": "get_time",
"parameters": {
"type": "object",
"properties": {"tz": {"type": "string"}},
},
},
)
]

tag = get_model_structural_tag(
model="hermes",
tools=tools,
tool_choice="auto",
reasoning=False,
force_strict=True,
)

assert isinstance(tag, StructuralTag)
rendered = str(tag)
assert "get_weather" in rendered
assert "get_time" in rendered
9 changes: 6 additions & 3 deletions vllm/envs.py
Original file line number Diff line number Diff line change
Expand Up @@ -220,7 +220,7 @@
MOONCAKE_REQUESTER_LOCAL_HOSTNAME: str | None = None
VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840
VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True
VLLM_ENFORCE_STRICT_TOOL_CALLING: bool | None = None
VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300
VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5
Expand Down Expand Up @@ -1668,9 +1668,12 @@ def _resolve_rust_cli_path() -> str | None:
"VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS": lambda: int(
os.getenv("VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS", "1")
),
# Enforce function parameter schemas in structural-tag based tool calling.
# Override structural-tag based tool calling. When unset, requests control
# strictness through the per-function `strict` field.
"VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: (
os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1")
None
if (value := os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING")) is None
else value.lower() in ("true", "1")
),
# Control the max chunk bytes (in MB) for the rpc message queue.
# Object larger than this threshold will be broadcast to worker
Expand Down
6 changes: 4 additions & 2 deletions vllm/tool_parsers/abstract_tool_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def __init_subclass__(cls, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
if (
cls.structural_tag_model is not None
and envs.VLLM_ENFORCE_STRICT_TOOL_CALLING
and envs.VLLM_ENFORCE_STRICT_TOOL_CALLING is not False
):
cls.supports_required_and_named = False

Expand Down Expand Up @@ -173,7 +173,8 @@ def get_structural_tag(
):
if self.structural_tag_model is None:
return None
if not envs.VLLM_ENFORCE_STRICT_TOOL_CALLING:
enforce_strict = envs.VLLM_ENFORCE_STRICT_TOOL_CALLING
if enforce_strict is False:
return None
from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag

Expand All @@ -182,6 +183,7 @@ def get_structural_tag(
tools=request.tools,
tool_choice=request.tool_choice,
reasoning=reasoning,
force_strict=enforce_strict is True,
)

def extract_tool_calls(
Expand Down
2 changes: 1 addition & 1 deletion vllm/tool_parsers/kimi_k3_tool_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ class KimiK3ToolParser(ToolParser):
supports_required_and_named = False
# Enables the vLLM-side XTML structural tag builder
# (``get_kimi_k3_structural_tag`` in ``structural_tag_registry``). With
# ``VLLM_ENFORCE_STRICT_TOOL_CALLING`` on (default), ``_apply_structural_tag``
# strict tool calling not explicitly disabled, ``_apply_structural_tag``
# constrains generation to K3's ``<|open|>tools<|sep|>`` channel for
# ``required`` (and ``auto`` when a tool sets ``strict``) instead of the
# generic JSON guided decoding, which conflicts with the XTML format.
Expand Down
26 changes: 21 additions & 5 deletions vllm/tool_parsers/structural_tag_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -105,16 +105,20 @@ def get_model_structural_tag(
tools: Sequence[ChatCompletionToolsParam | ResponsesTool] | None,
tool_choice: ToolChoice,
reasoning: bool,
*,
force_strict: bool = False,
) -> StructuralTag | None:
"""Build a structural tag with xgrammar's builtin model templates."""

if not tools or tool_choice == "none":
return None

if tool_choice == "auto" and not _any_tool_strict(tools):
if tool_choice == "auto" and not force_strict and not _any_tool_strict(tools):
return None

dumped_tools = [_dump_tool_for_xgrammar(tool) for tool in tools]
dumped_tools = [
_dump_tool_for_xgrammar(tool, force_strict=force_strict) for tool in tools
]
dumped_tool_choice = _dump_tool_choice_for_xgrammar(tool_choice)

if model in _VLLM_STRUCTURAL_TAG_REGISTRY:
Expand Down Expand Up @@ -143,20 +147,32 @@ def get_model_structural_tag(

def _dump_tool_for_xgrammar(
tool: ChatCompletionToolsParam | ResponsesTool,
*,
force_strict: bool = False,
) -> dict[str, Any]:
"""Convert tool objects to xgrammar's Chat Completions tool protocol."""
"""Convert tool objects to xgrammar's Chat Completions tool protocol.

Args:
tool: The requested tool.
force_strict: Whether to mark the dumped copy as strict without
mutating the request.
"""

if isinstance(tool, FunctionTool):
function: dict[str, Any] = {"name": tool.name}
if tool.description is not None:
function["description"] = tool.description
if tool.parameters is not None:
function["parameters"] = tool.parameters
if tool.strict is not None:
function["strict"] = tool.strict
strict = True if force_strict else tool.strict
if strict is not None:
function["strict"] = strict
return {"type": "function", "function": function}
dumped_tool = tool.model_dump(mode="json", exclude_none=True)
if isinstance(tool, ChatCompletionToolsParam):
strict = True if force_strict else tool.function.strict
if strict is not None:
dumped_tool["function"]["strict"] = strict
return dumped_tool
return dict(dumped_tool)

Expand Down
Loading