From 119153787464fca3697770b9cecc0f8dcdae9388 Mon Sep 17 00:00:00 2001 From: lazymio Date: Thu, 10 Sep 2026 12:07:11 +0000 Subject: [PATCH 1/5] [Frontend] Add VLLM_TOOL_STRICT_LEVEL to raise the tool-call grammar floor get_model_structural_tag() drops the structural tag for tool_choice="auto" unless some tool declares strict. That is a client-side decision, and the clients that matter never make it: across 9,500 production requests to a DeepSeek-V4-Flash deployment, tool_choice was absent on every one and strict was false on all 17,476 tool definitions, which is what the OpenAI SDKs and common agent harnesses emit by default. A server operator has no way to constrain the tool-call envelope for that traffic, so the model is free to emit malformed markup no parser can recover. Add a server-side floor, mirroring SGLANG_TOOL_STRICT_LEVEL: off only tools marked strict constrain an "auto" request (default, behaviour unchanged) function constrain the tool-call envelope for every request with tools parameter additionally pin argument schemas, as if every tool were strict "function" pins the envelope (markup, function name from the declared tools, parameter tag shape) while argument contents stay free. xgrammar treats an unset strict as "constrain the arguments", so the level marks those tools non-strict explicitly on a copy; the request's tools are never mutated. The level is a floor: it only lifts the auto + non-strict gate and never relaxes required / named tool choice or tools the client marked strict. It does not force a tool call either, since the builtin tags only engage once the model opens the wrapper itself. Unknown values warn once and fall back to "off" so a typo cannot take a server down at request time. VLLM_ENFORCE_STRICT_TOOL_CALLING=false still disables structural tags entirely. Co-Authored-By: Claude Opus 5 (1M context) Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01ATF7hTE4aSLHerdbvktFwu Signed-off-by: lazymio --- docs/features/tool_calling.md | 20 ++- .../test_structural_tag_registry.py | 164 ++++++++++++++++++ vllm/envs.py | 7 + vllm/tool_parsers/structural_tag_registry.py | 88 +++++++++- 4 files changed, 270 insertions(+), 9 deletions(-) diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 9628bc431a1e..0f7061523ee0 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -115,7 +115,7 @@ Whether vLLM enforces the tool parameter schema during generation depends on the | --- | --- | --- | | Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. | | `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. | -| `"auto"` | Only when `strict: true` is set on at least one tool | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`. Without it, the model generates freely and tool calls are extracted from raw text. | +| `"auto"` | Only when `strict: true` is set on at least one tool (or the server raises the floor via `VLLM_TOOL_STRICT_LEVEL`) | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`. Without it, the model generates freely and tool calls are extracted from raw text. | | `"none"` | N/A | No tool calls are produced. | ### Strict Mode @@ -134,6 +134,22 @@ vLLM also provides a global toggle via the `VLLM_ENFORCE_STRICT_TOOL_CALLING` en VLLM_ENFORCE_STRICT_TOOL_CALLING=false vllm serve ... ``` +### Server-Side Strictness Floor + +Most OpenAI-compatible clients and agent frameworks never set `strict` on their tools, so with `tool_choice="auto"` the model generates tool calls without any grammar and malformed markup can leak into the response. The `VLLM_TOOL_STRICT_LEVEL` environment variable lets the server operator raise the floor for every request that carries tools, independently of what the client declares (similar to SGLang's `SGLANG_TOOL_STRICT_LEVEL`): + +| Value | Behavior | +| --- | --- | +| `off` (default) | Unchanged: with `tool_choice="auto"`, structural tags apply only when a tool sets `strict: true`. | +| `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. | +| `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. | + +```bash +VLLM_TOOL_STRICT_LEVEL=function vllm serve ... +``` + +The floor never relaxes a constraint the request would already receive: `tool_choice="required"` and named function calling keep their schema-derived constraints at every level, and tools the client marked `strict: true` keep their schemas. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this variable. + ## Automatic Function Calling To enable this feature, you should set the following flags: @@ -152,7 +168,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template! !!! note - With `tool_choice="auto"`, schema-level constraint requires both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`. When these conditions are met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. + With `tool_choice="auto"`, schema-level constraint requires both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `VLLM_TOOL_STRICT_LEVEL`. When these conditions are met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. ### Hermes Models (`hermes`) diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 06c8fb61295c..9e16bbab2d03 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json from types import SimpleNamespace from unittest.mock import MagicMock @@ -8,6 +9,7 @@ from xgrammar import Grammar, StructuralTag, builtin_structural_tag from xgrammar.testing import _is_grammar_accept_string +from vllm import envs from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedFunction, ChatCompletionNamedToolChoiceParam, @@ -33,6 +35,7 @@ SUPPORTED_STRUCTURAL_TAG_MODELS, VLLM_BUILTIN_STRUCTURAL_TAG_MODELS, XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS, + ToolChoice, get_function_parameters, get_model_structural_tag, ) @@ -829,3 +832,164 @@ def test_kimi_k3_forced_tool_choice_builds_single_mandatory_call(): response_only = _k3_response("no call here") assert _is_grammar_accept_string(grammar, ok) assert not _is_grammar_accept_string(grammar, response_only) + + +def _pins_argument_schema(tag: StructuralTag) -> bool: + return '"json_schema": {' in json.dumps(tag.model_dump(), ensure_ascii=False) + + +def test_tool_strict_level_off_is_the_default( + sample_tools: list[ChatCompletionToolsParam], +): + """auto + no strict tool still gets no tag unless the operator opts in.""" + assert envs.VLLM_TOOL_STRICT_LEVEL == "off" + assert ( + get_model_structural_tag( + model="deepseek_v4", + tools=sample_tools, + tool_choice="auto", + reasoning=False, + ) + is None + ) + + +@pytest.mark.parametrize("model", sorted(SUPPORTED_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_function_lifts_auto_gate( + model: str, + sample_tools: list[ChatCompletionToolsParam], + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "function", raising=False) + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + ) + + assert tag is not None + + +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_function_pins_envelope_only( + model: str, + sample_tools: list[ChatCompletionToolsParam], + monkeypatch: pytest.MonkeyPatch, +): + """xgrammar constrains arguments for tools whose ``strict`` is unset, so + the level must mark them non-strict explicitly or it would behave like + "parameter". The request's own tools stay untouched.""" + monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "function", raising=False) + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + ) + + assert tag is not None + assert not _pins_argument_schema(tag) + assert all(tool.function.strict is None for tool in sample_tools) + + +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_tool_strict_level_parameter_pins_argument_schemas( + model: str, + sample_tools: list[ChatCompletionToolsParam], + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "parameter", raising=False) + tag = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice="auto", + reasoning=False, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +def test_tool_strict_level_function_keeps_client_strict_tools_strict( + sample_tools_strict: list[ChatCompletionToolsParam], + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "function", raising=False) + tag = get_model_structural_tag( + model="deepseek_v4", + tools=sample_tools_strict, + tool_choice="auto", + reasoning=False, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +@pytest.mark.parametrize( + "tool_choice", + [ + "required", + ChatCompletionNamedToolChoiceParam( + function=ChatCompletionNamedFunction(name="get_weather") + ), + ], +) +def test_tool_strict_level_function_never_relaxes_forced_tool_choice( + tool_choice: ToolChoice, + sample_tools: list[ChatCompletionToolsParam], + monkeypatch: pytest.MonkeyPatch, +): + """The level is a floor: required / named calls keep their schema pinning.""" + monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "function", raising=False) + tag = get_model_structural_tag( + model="deepseek_v4", + tools=sample_tools, + tool_choice=tool_choice, + reasoning=False, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +def test_tool_strict_level_parameter_overrides_client_strict_false( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "parameter", raising=False) + tools = [ + ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + "strict": False, + }, + ) + ] + tag = get_model_structural_tag( + model="deepseek_v4", + tools=tools, + tool_choice="required", + reasoning=False, + ) + + assert tag is not None + assert _pins_argument_schema(tag) + + +def test_tool_strict_level_unknown_value_falls_back_to_off( + sample_tools: list[ChatCompletionToolsParam], + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "nonsense", raising=False) + assert ( + get_model_structural_tag( + model="deepseek_v4", + tools=sample_tools, + tool_choice="auto", + reasoning=False, + ) + is None + ) diff --git a/vllm/envs.py b/vllm/envs.py index c512df457adc..123a72ebf1a9 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -239,6 +239,7 @@ VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840 VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1 VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True + VLLM_TOOL_STRICT_LEVEL: str = "off" VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300 VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5 @@ -1759,6 +1760,12 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: ( os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1") ), + # Server-side floor for structural-tag based tool calling, applied on top + # of the per-tool `strict` field: "off" (default) only constrains an + # "auto" request when a tool sets strict, "function" constrains the + # tool-call envelope for every request with tools, "parameter" also pins + # argument schemas. Mirrors SGLANG_TOOL_STRICT_LEVEL. + "VLLM_TOOL_STRICT_LEVEL": lambda: os.getenv("VLLM_TOOL_STRICT_LEVEL", "off"), # Control the max chunk bytes (in MB) for the rpc message queue. # Object larger than this threshold will be broadcast to worker # processes via zmq. diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 295d5844b430..81d8752f99ab 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import enum from collections.abc import Callable, Sequence from typing import Any, Literal, TypeAlias @@ -31,10 +32,14 @@ TriggeredTagsFormat, ) +from vllm import envs from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, ChatCompletionToolsParam, ) +from vllm.logger import init_logger + +logger = init_logger(__name__) ToolChoice: TypeAlias = ( Literal["none", "auto", "required"] @@ -94,15 +99,83 @@ def decorator(func: StructuralTagBuilder) -> StructuralTagBuilder: return decorator +class ToolStrictLevel(enum.IntEnum): + """Server-side floor for tool-call structural tags (VLLM_TOOL_STRICT_LEVEL). + + OFF: only tools the client marked ``strict`` constrain an "auto" + request (default). + FUNCTION: constrain the tool-call envelope for every request with tools. + PARAMETER: additionally pin argument schemas, as if every tool were + ``strict``. + """ + + OFF = 0 + FUNCTION = 1 + PARAMETER = 2 + + +def _tool_strict_level() -> ToolStrictLevel: + raw = envs.VLLM_TOOL_STRICT_LEVEL.strip().lower() + try: + return ToolStrictLevel[raw.upper()] + except KeyError: + logger.warning_once( + "Unknown VLLM_TOOL_STRICT_LEVEL %r; expected one of %s. Using 'off'.", + raw, + ", ".join(level.name.lower() for level in ToolStrictLevel), + ) + return ToolStrictLevel.OFF + + +def _tool_is_strict(tool: ChatCompletionToolsParam | ResponsesTool) -> bool: + if isinstance(tool, FunctionTool): + return tool.strict is True + if isinstance(tool, ChatCompletionToolsParam): + return tool.function.strict is True + return False + + def _any_tool_strict( tools: Sequence[ChatCompletionToolsParam | ResponsesTool], ) -> bool: - for tool in tools: - if isinstance(tool, FunctionTool) and tool.strict is True: - return True - if isinstance(tool, ChatCompletionToolsParam) and tool.function.strict is True: - return True - return False + return any(_tool_is_strict(tool) for tool in tools) + + +def _with_tool_strict( + tool: ChatCompletionToolsParam | ResponsesTool, + strict: bool, +) -> ChatCompletionToolsParam | ResponsesTool: + """Return a copy of ``tool`` with ``strict`` set, leaving the request's alone.""" + if isinstance(tool, FunctionTool): + return tool.model_copy(update={"strict": strict}) + if isinstance(tool, ChatCompletionToolsParam): + return tool.model_copy( + update={"function": tool.function.model_copy(update={"strict": strict})} + ) + return tool + + +def _apply_tool_strict_level( + tools: Sequence[ChatCompletionToolsParam | ResponsesTool], + tool_choice: ToolChoice, +) -> Sequence[ChatCompletionToolsParam | ResponsesTool] | None: + """Apply VLLM_TOOL_STRICT_LEVEL; ``None`` means no structural tag.""" + level = _tool_strict_level() + + if tool_choice == "auto" and not _any_tool_strict(tools): + if level < ToolStrictLevel.FUNCTION: + return None + if level == ToolStrictLevel.FUNCTION: + # xgrammar pins argument schemas for tools whose ``strict`` is + # unset, so mark them non-strict to constrain the envelope only. + return [_with_tool_strict(tool, False) for tool in tools] + + if level >= ToolStrictLevel.PARAMETER: + return [ + tool if _tool_is_strict(tool) else _with_tool_strict(tool, True) + for tool in tools + ] + return tools def get_model_structural_tag( @@ -116,7 +189,8 @@ def get_model_structural_tag( if not tools or tool_choice == "none": return None - if tool_choice == "auto" and not _any_tool_strict(tools): + tools = _apply_tool_strict_level(tools, tool_choice) + if tools is None: return None dumped_tools = [_dump_tool_for_xgrammar(tool) for tool in tools] From ed1ba1964afe9c669b53dfc768e87bcaaaae0583 Mon Sep 17 00:00:00 2001 From: lazymio Date: Thu, 17 Sep 2026 12:36:58 +0000 Subject: [PATCH 2/5] Update according to maintainer feedback --- docs/features/tool_calling.md | 18 +- rust/src/chat/src/backend/hf.rs | 2 + rust/src/chat/src/backend/mod.rs | 3 +- rust/src/chat/src/lib.rs | 19 +- rust/src/chat/src/output/default/mod.rs | 15 +- .../chat/src/output/default/structural_tag.rs | 210 +++++++++++++++--- rust/src/chat/src/parser/mod.rs | 60 +++++ rust/src/chat/tests/chat.rs | 1 + rust/src/chat/tests/roundtrip.rs | 1 + rust/src/cmd/src/cli.rs | 16 ++ rust/src/cmd/src/cli/tests.rs | 6 + rust/src/server/src/config.rs | 3 + rust/src/server/src/grpc/tests.rs | 2 + rust/src/server/src/lib.rs | 3 +- rust/src/server/src/render.rs | 14 +- .../server/src/routes/http_client_tests.rs | 1 + rust/src/server/src/routes/tests.rs | 1 + .../test_structural_tag_registry.py | 167 ++++++++++---- vllm/entrypoints/anthropic/serving.py | 2 + vllm/entrypoints/cohere/serving.py | 2 + vllm/entrypoints/generate/api_router.py | 4 + .../launchers/api_server/app_state.py | 2 + vllm/entrypoints/launchers/cli_args.py | 8 + .../entrypoints/launchers/render/app_state.py | 2 + .../openai/chat_completion/serving.py | 2 + vllm/entrypoints/openai/responses/serving.py | 2 + vllm/envs.py | 7 - vllm/parser/abstract_parser.py | 4 + vllm/parser/parser_manager.py | 10 + vllm/renderers/online_derenderer.py | 2 + vllm/renderers/online_renderer.py | 2 + vllm/tool_parsers/abstract_tool_parser.py | 3 + vllm/tool_parsers/hy_v4_tool_parser.py | 4 + vllm/tool_parsers/structural_tag_registry.py | 75 ++----- vllm/tool_parsers/tool_strict_level.py | 32 +++ 35 files changed, 553 insertions(+), 152 deletions(-) create mode 100644 vllm/tool_parsers/tool_strict_level.py diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 0f7061523ee0..2ae356a0c6ba 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -109,18 +109,18 @@ vLLM supports the `tool_choice='none'` option in the chat completion API. When t ## Constrained Decoding Behavior -Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode and the per-tool `strict` field: +Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode, the per-tool `strict` field, and the server-side `--tool-strict-level`: | `tool_choice` value | Schema-constrained decoding | Behavior | | --- | --- | --- | -| Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. | +| Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. With a structural-tag parser, the call is always constrained but the argument schema is pinned only for tools with `strict: true` (or `--tool-strict-level parameter`). | | `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. | -| `"auto"` | Only when `strict: true` is set on at least one tool (or the server raises the floor via `VLLM_TOOL_STRICT_LEVEL`) | Structural-tag parsers constrain tool-call arguments when a tool opts in with `strict: true`. Without it, the model generates freely and tool calls are extracted from raw text. | +| `"auto"` | Only when `strict: true` is set on at least one tool, or the server raises the floor via `--tool-strict-level` | Structural-tag parsers constrain the call envelope, and pin the argument schema of each tool that opts in with `strict: true`. Without either, the model generates freely and tool calls are extracted from raw text. | | `"none"` | N/A | No tool calls are produced. | ### Strict Mode -For `tool_choice="required"` or named function calling, structural-tag constraints are always applied regardless of the `strict` field. For `tool_choice="auto"`, setting `strict: true` on at least one tool opts in to structural-tag constraints; without it, the model generates freely and tool calls are extracted from raw text. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages. +Structural-tag constraints have two layers. The tool-call *envelope* (markup and function name) is constrained whenever a structural tag applies: always for `tool_choice="required"` and named function calling, and for `tool_choice="auto"` once at least one tool sets `strict: true` or the server raises the floor. The *argument schema* of an individual tool is pinned only when that tool sets `strict: true` (or under `--tool-strict-level parameter`); a tool that omits `strict` keeps its arguments unconstrained, even when another tool in the same request is strict. The `strict` field is supported across all three API surfaces: Chat Completion, Responses, and Anthropic Messages. For best compatibility with strict schema enforcement, define tool parameter schemas in the OpenAI strict-schema style: @@ -136,19 +136,19 @@ VLLM_ENFORCE_STRICT_TOOL_CALLING=false vllm serve ... ### Server-Side Strictness Floor -Most OpenAI-compatible clients and agent frameworks never set `strict` on their tools, so with `tool_choice="auto"` the model generates tool calls without any grammar and malformed markup can leak into the response. The `VLLM_TOOL_STRICT_LEVEL` environment variable lets the server operator raise the floor for every request that carries tools, independently of what the client declares (similar to SGLang's `SGLANG_TOOL_STRICT_LEVEL`): +Most OpenAI-compatible clients and agent frameworks never set `strict` on their tools, so with `tool_choice="auto"` the model generates tool calls without any grammar and malformed markup can leak into the response. The `--tool-strict-level` option lets the server operator raise the floor for every request that carries tools, independently of what the client declares (similar to SGLang's `SGLANG_TOOL_STRICT_LEVEL`): | Value | Behavior | | --- | --- | -| `off` (default) | Unchanged: with `tool_choice="auto"`, structural tags apply only when a tool sets `strict: true`. | +| `off` (default) | With `tool_choice="auto"`, structural tags apply only when a tool sets `strict: true`. | | `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. | | `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. | ```bash -VLLM_TOOL_STRICT_LEVEL=function vllm serve ... +vllm serve ... --tool-strict-level function ``` -The floor never relaxes a constraint the request would already receive: `tool_choice="required"` and named function calling keep their schema-derived constraints at every level, and tools the client marked `strict: true` keep their schemas. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this variable. +The floor never relaxes a constraint the request would already receive: tools the client marked `strict: true` keep their schemas at every level. With `tool_choice="auto"`, the grammar does not force a tool call; a plain text response stays valid. `VLLM_ENFORCE_STRICT_TOOL_CALLING=false` disables structural tags entirely and takes precedence over this option. ## Automatic Function Calling @@ -168,7 +168,7 @@ from HuggingFace; and you can find an example of this in a `tokenizer_config.jso If your favorite tool-calling model is not supported, please feel free to contribute a parser & tool use chat template! !!! note - With `tool_choice="auto"`, schema-level constraint requires both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `VLLM_TOOL_STRICT_LEVEL`. When these conditions are met and the selected parser supports structural tags, vLLM constrains tool-call arguments. Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. + With `tool_choice="auto"`, structural-tag constraints require both `VLLM_ENFORCE_STRICT_TOOL_CALLING=true` (the default) and at least one tool with `strict: true`, or a server-side floor set via `--tool-strict-level`. When these conditions are met and the selected parser supports structural tags, vLLM constrains the tool-call envelope and pins the argument schema of each tool that sets `strict: true` (or of every tool under `--tool-strict-level parameter`). Otherwise, vLLM extracts tool calls from raw text, so arguments may occasionally be malformed or violate the function's parameter schema. ### Hermes Models (`hermes`) diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index af9af8b51ca4..d16f37a8911b 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -134,6 +134,7 @@ impl ChatBackend for HfChatBackend { self.tokenizer.clone(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } @@ -343,6 +344,7 @@ mod tests { let error = match backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { + tool_strict_level: crate::ToolStrictLevel::Off, tool_call_parser: &ParserSelection::Explicit("json".to_string()), reasoning_parser: &ParserSelection::Auto, }, diff --git a/rust/src/chat/src/backend/mod.rs b/rust/src/chat/src/backend/mod.rs index af4579159e82..a0343e53f327 100644 --- a/rust/src/chat/src/backend/mod.rs +++ b/rust/src/chat/src/backend/mod.rs @@ -13,7 +13,7 @@ use crate::multimodal::{MmLimitPerPrompt, MultimodalModelInfo}; use crate::output::DynChatOutputProcessor; use crate::renderer::DynChatRenderer; use crate::request::ChatRequest; -use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; +use crate::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection, ToolStrictLevel}; pub mod hf; @@ -21,6 +21,7 @@ pub mod hf; pub struct NewChatOutputProcessorOptions<'a> { pub tool_call_parser: &'a ParserSelection, pub reasoning_parser: &'a ParserSelection, + pub tool_strict_level: ToolStrictLevel, } /// Minimal prompt-processing backend needed by `vllm-chat`. diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 69d39ca78ed3..42bb5633d027 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -30,7 +30,7 @@ pub use parser::reasoning::{ ReasoningDelta, ReasoningError, ReasoningParser, ReasoningParserFactory, }; pub use parser::tool::{ToolParser, ToolParserError, ToolParserFactory}; -pub use parser::{ParserSelection, validate_parser_overrides}; +pub use parser::{ParserSelection, ToolStrictLevel, validate_parser_overrides}; pub use reasoning::EffortValue; pub use renderer::hf::ChatTemplateContentFormatOption; pub use renderer::{ @@ -74,6 +74,8 @@ pub struct ChatRequestProcessor { tool_call_parser: ParserSelection, /// Reasoning parser selection used when preparing generation requests. reasoning_parser: ParserSelection, + /// Server-side floor for tool-call structural tags. + tool_strict_level: ToolStrictLevel, } impl ChatRequestProcessor { @@ -85,6 +87,7 @@ impl ChatRequestProcessor { model_dtype: Some(model_dtype), tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Off, } } @@ -95,6 +98,7 @@ impl ChatRequestProcessor { model_dtype: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Off, } } @@ -109,6 +113,12 @@ impl ChatRequestProcessor { self } + /// Configure the server-side floor for tool-call structural tags. + pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self { + self.tool_strict_level = tool_strict_level; + self + } + async fn finalize_rendered_prompt( &self, request: &ChatRequest, @@ -207,6 +217,7 @@ impl ChatRequestProcessor { NewChatOutputProcessorOptions { tool_call_parser: &self.tool_call_parser, reasoning_parser: &self.reasoning_parser, + tool_strict_level: self.tool_strict_level, }, )?; let text_request = self.prepare_text_request(request).await?; @@ -255,6 +266,12 @@ impl ChatLlm { self } + /// Set the server-side floor for tool-call structural tags. + pub fn with_tool_strict_level(mut self, tool_strict_level: ToolStrictLevel) -> Self { + self.processor.tool_strict_level = tool_strict_level; + self + } + /// Override the effective model dtype used for multimodal tensor encoding. pub fn with_model_dtype(mut self, model_dtype: ModelDtype) -> Self { self.processor.model_dtype = Some(model_dtype); diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index a73115306342..0019d431941b 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -18,10 +18,10 @@ use self::unified::unified_event_stream; use super::structured::structured_chat_event_stream; use crate::error::Result; use crate::output::{ChatOutputProcessor, DynChatEventStream, DynDecodedTextEventStream}; -use crate::parser::ParserSelection; use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; use crate::parser::unified::UnifiedParserFactory; +use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::{ChatRequest, ChatTool}; use crate::{Error, Result as ChatResult}; @@ -49,6 +49,7 @@ impl DefaultChatOutputProcessor { tokenizer: DynTokenizer, tool_call_parser: &ParserSelection, reasoning_parser: &ParserSelection, + tool_strict_level: ToolStrictLevel, ) -> ChatResult { let parser = if let Some(parser) = Self::resolve_optional_unified_parser( request.tools(), @@ -73,7 +74,11 @@ impl DefaultChatOutputProcessor { Box::new(CombinedParser::new(reasoning_parser, tool_parser)) as Box }; - apply_structural_tag_constraint(request, parser.structural_tag_builder())?; + apply_structural_tag_constraint( + request, + parser.structural_tag_builder(), + tool_strict_level, + )?; if parser.preserve_special_tokens() { request.decode_options.skip_special_tokens = false; @@ -191,7 +196,7 @@ mod tests { use vllm_tokenizer::test_utils::TestTokenizer; use super::DefaultChatOutputProcessor; - use crate::parser::ParserSelection; + use crate::parser::{ParserSelection, ToolStrictLevel}; use crate::request::ChatRequest; fn tokenizer() -> Arc { @@ -213,6 +218,7 @@ mod tests { tokenizer(), &selection, &selection, + ToolStrictLevel::Off, ) .unwrap(); } @@ -227,6 +233,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Auto, + ToolStrictLevel::Off, ) .unwrap(); } @@ -244,6 +251,7 @@ mod tests { tokenizer(), tool, reasoning, + ToolStrictLevel::Off, ) .unwrap(); } @@ -258,6 +266,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Explicit("gemma4".to_string()), + ToolStrictLevel::Off, ) { Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"), Err(error) => error, diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs index 2ef6f0868593..c4eb285600b4 100644 --- a/rust/src/chat/src/output/default/structural_tag.rs +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -14,19 +14,21 @@ use xgrammar_structural_tag::{ build_structural_tag, }; +use crate::parser::ToolStrictLevel; use crate::request::{ChatRequest, ChatToolChoice}; use crate::{Error, Result as ChatResult}; /// Apply structural tag constraints to the request based on the tool parser's structural tag -/// support and the request's tool choice. +/// support, the request's tool choice, and the server-side strictness floor. pub(super) fn apply_structural_tag_constraint( request: &mut ChatRequest, builder: Option<&dyn StructuralTagBuilder>, + strict_level: ToolStrictLevel, ) -> ChatResult<()> { let Some(builder) = builder else { return Ok(()); }; - let Some(tool_choice) = structural_tag_tool_choice(request) else { + let Some(tool_choice) = structural_tag_tool_choice(request, strict_level) else { return Ok(()); }; @@ -34,11 +36,14 @@ pub(super) fn apply_structural_tag_constraint( .tools() .iter() .map(|tool| { + // A tool without an explicit `strict` is non-strict: its call envelope is + // constrained, but its arguments stay free unless the server level is `parameter`. + let strict = strict_level >= ToolStrictLevel::Parameter || tool.strict == Some(true); ToolParam::Function(FunctionToolParam::new(FunctionDefinition { name: tool.name.clone(), description: tool.description.clone(), parameters: Some(tool.parameters.clone()), - strict: tool.strict, + strict: Some(strict), })) }) .collect::>(); @@ -66,17 +71,21 @@ pub(super) fn apply_structural_tag_constraint( /// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request. /// /// Returns `None` if no structural tag constraints should be applied. -fn structural_tag_tool_choice(request: &ChatRequest) -> Option { +fn structural_tag_tool_choice( + request: &ChatRequest, + strict_level: ToolStrictLevel, +) -> Option { if request.tools().is_empty() { return None; } match request.tool_choice() { - // For `Auto`, only apply the structural tag if there's at least one strict tool. - ChatToolChoice::Auto if request.tools().iter().any(|tool| tool.strict == Some(true)) => { - Some(StructuralTagToolChoice::auto()) - } - ChatToolChoice::Auto | ChatToolChoice::None => None, + ChatToolChoice::None => None, + // For `Auto`, apply the structural tag only when the server raises the floor or at + // least one tool opts in with `strict: true`. + ChatToolChoice::Auto => (strict_level >= ToolStrictLevel::Function + || request.tools().iter().any(|tool| tool.strict == Some(true))) + .then(StructuralTagToolChoice::auto), ChatToolChoice::Required => Some(StructuralTagToolChoice::required()), ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())), @@ -148,8 +157,12 @@ mod tests { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert_eq!(tag["type"], "structural_tag"); @@ -172,8 +185,12 @@ mod tests { }; let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert!(tag.to_string().contains("lookup")); @@ -199,8 +216,12 @@ mod tests { }; let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request).to_string(); assert!(tag.contains("search")); @@ -212,8 +233,12 @@ mod tests { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag decision should succeed"); assert!(request.sampling_params.structured_outputs.is_none()); } @@ -227,8 +252,12 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag should build"); let params = structured_outputs(&request); assert!(params.constraint.is_structural_tag()); @@ -242,8 +271,12 @@ mod tests { let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request); assert_eq!(tag["type"], "structural_tag"); @@ -259,8 +292,12 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag should build"); let params = structured_outputs(&request); assert!(params.constraint.is_structural_tag()); @@ -279,8 +316,12 @@ mod tests { ); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag should build"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag should build"); let tag = structural_tag_value(&request).to_string(); assert!(tag.contains("lookup")); @@ -292,8 +333,12 @@ mod tests { let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag decision should succeed"); assert!(request.sampling_params.structured_outputs.is_none()); } @@ -307,10 +352,117 @@ mod tests { }); let parser = qwen3_coder_parser(request.tools()); - apply_structural_tag_constraint(&mut request, parser.structural_tag_builder()) - .expect("structural tag decision should succeed"); + apply_structural_tag_constraint( + &mut request, + parser.structural_tag_builder(), + ToolStrictLevel::Off, + ) + .expect("structural tag decision should succeed"); let params = structured_outputs(&request); assert!(params.constraint.is_json_object()); } + + fn build(tool_choice: ChatToolChoice, tools: Vec, level: ToolStrictLevel) -> ChatRequest { + let mut request = request(tool_choice, tools); + let parser = qwen3_coder_parser(request.tools()); + + apply_structural_tag_constraint(&mut request, parser.structural_tag_builder(), level) + .expect("structural tag decision should succeed"); + request + } + + fn tag_string(request: &ChatRequest) -> String { + structural_tag_value(request).to_string() + } + + #[test] + fn function_level_constrains_auto_without_strict_tools() { + let request = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Function, + ); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn absent_strict_is_non_strict() { + let unset = build( + ChatToolChoice::Required, + vec![chat_tool("search", None)], + ToolStrictLevel::Off, + ); + let non_strict = build( + ChatToolChoice::Required, + vec![chat_tool("search", Some(false))], + ToolStrictLevel::Off, + ); + let strict = build( + ChatToolChoice::Required, + vec![chat_tool("search", Some(true))], + ToolStrictLevel::Off, + ); + + assert_eq!(tag_string(&unset), tag_string(&non_strict)); + assert_ne!(tag_string(&unset), tag_string(&strict)); + } + + #[test] + fn strict_tool_does_not_pin_its_neighbour() { + let mixed = build( + ChatToolChoice::Auto, + vec![chat_tool("weather", Some(true)), chat_tool("search", None)], + ToolStrictLevel::Off, + ); + let explicit = build( + ChatToolChoice::Auto, + vec![ + chat_tool("weather", Some(true)), + chat_tool("search", Some(false)), + ], + ToolStrictLevel::Off, + ); + + assert_eq!(tag_string(&mixed), tag_string(&explicit)); + } + + #[test] + fn parameter_level_pins_argument_schemas() { + let pinned = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Parameter, + ); + let strict = build( + ChatToolChoice::Auto, + vec![chat_tool("search", Some(true))], + ToolStrictLevel::Off, + ); + let envelope_only = build( + ChatToolChoice::Auto, + vec![chat_tool("search", None)], + ToolStrictLevel::Function, + ); + + assert_eq!(tag_string(&pinned), tag_string(&strict)); + assert_ne!(tag_string(&pinned), tag_string(&envelope_only)); + } + + #[test] + fn tool_strict_level_parses_case_insensitively() { + assert_eq!( + "PARAMETER".parse::(), + Ok(ToolStrictLevel::Parameter) + ); + assert_eq!( + "function".parse::(), + Ok(ToolStrictLevel::Function) + ); + assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Off); + assert!("strict".parse::().is_err()); + } } diff --git a/rust/src/chat/src/parser/mod.rs b/rust/src/chat/src/parser/mod.rs index c10a4f11bf3e..c42835b71e05 100644 --- a/rust/src/chat/src/parser/mod.rs +++ b/rust/src/chat/src/parser/mod.rs @@ -57,6 +57,66 @@ impl ParserSelection { } } +/// Server-side floor for tool-call structural tags (`--tool-strict-level`). +#[derive( + Debug, + Clone, + Copy, + PartialEq, + Eq, + PartialOrd, + Ord, + Default, + DeserializeFromStr, + SerializeDisplay, +)] +pub enum ToolStrictLevel { + /// Constrain a `tool_choice = "auto"` request only when a tool sets `strict: true`. + #[default] + Off, + /// Constrain the tool-call envelope for every request with tools. + Function, + /// Additionally pin argument schemas, as if every tool were `strict: true`. + Parameter, +} + +impl ToolStrictLevel { + pub const OFF_LITERAL: &str = "off"; + pub const FUNCTION_LITERAL: &str = "function"; + pub const PARAMETER_LITERAL: &str = "parameter"; +} + +impl FromStr for ToolStrictLevel { + type Err = String; + + fn from_str(value: &str) -> Result { + if value.eq_ignore_ascii_case(Self::OFF_LITERAL) { + Ok(Self::Off) + } else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) { + Ok(Self::Function) + } else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) { + Ok(Self::Parameter) + } else { + Err(format!( + "unknown tool strict level {value:?}; expected one of {}, {}, {}", + Self::OFF_LITERAL, + Self::FUNCTION_LITERAL, + Self::PARAMETER_LITERAL + )) + } + } +} + +impl fmt::Display for ToolStrictLevel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::Off => Self::OFF_LITERAL, + Self::Function => Self::FUNCTION_LITERAL, + Self::Parameter => Self::PARAMETER_LITERAL, + }) + } +} + /// Validate explicit parser override names without starting request processing. pub fn validate_parser_overrides( tool_call_parser: &ParserSelection, diff --git a/rust/src/chat/tests/chat.rs b/rust/src/chat/tests/chat.rs index ed96134cdc23..ab366b8827a4 100644 --- a/rust/src/chat/tests/chat.rs +++ b/rust/src/chat/tests/chat.rs @@ -227,6 +227,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 5669e255e0d5..4557f5fdd0da 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -717,6 +717,7 @@ async fn parse_completion( let processor = backends.chat_backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { + tool_strict_level: vllm_chat::ToolStrictLevel::Off, tool_call_parser: &case.tool_call_parser, reasoning_parser: &case.reasoning_parser, }, diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index cda2e6c1a8b9..7b999b7047c1 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -24,6 +24,7 @@ use serde_with::{DefaultOnNull, OneOrMany, serde_as}; use thiserror_ext::AsReport as _; use uuid::Uuid; use vllm_chat::GenerationConfigMode; +use vllm_chat::ToolStrictLevel; use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_engine_core_client::TransportMode; use vllm_managed_engine::ManagedEngineConfig; @@ -134,6 +135,10 @@ pub struct RenderArgs { /// `none` to disable parsing. #[arg(long, default_value_t)] reasoning_parser: ParserSelection, + /// Server-side floor for structural-tag based tool calling: `off`, + /// `function`, or `parameter`. + #[arg(long, default_value_t)] + tool_strict_level: ToolStrictLevel, /// Select the native chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] renderer: RendererSelection, @@ -171,6 +176,7 @@ impl RenderArgs { port: self.port, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, chat_template: self.chat_template, default_chat_template_kwargs: self.default_chat_template_kwargs.unwrap_or_default(), @@ -241,6 +247,14 @@ pub struct SharedRuntimeArgs { #[arg(long, default_value_t)] #[serde(default = "default_py_bootstrap_parser_selection")] pub reasoning_parser: ParserSelection, + /// Server-side floor for structural-tag based tool calling, applied on + /// top of the per-tool `strict` field: `off` constrains a + /// `tool_choice = "auto"` request only when a tool sets `strict: true`, + /// `function` constrains the tool-call envelope for every request with + /// tools, `parameter` additionally pins argument schemas. + #[arg(long, default_value_t)] + #[serde(default)] + pub tool_strict_level: ToolStrictLevel, /// Select the chat renderer implementation. #[arg(long = "tokenizer-mode", default_value_t)] #[serde(default, rename = "tokenizer_mode")] @@ -517,6 +531,7 @@ impl SharedRuntimeArgs { listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd }, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, language_model_only: self.language_model_only, chat_template: self.chat_template, @@ -574,6 +589,7 @@ impl SharedRuntimeArgs { listener_mode, tool_call_parser: self.tool_call_parser, reasoning_parser: self.reasoning_parser, + tool_strict_level: self.tool_strict_level, renderer: self.renderer, language_model_only: self.language_model_only, chat_template: self.chat_template, diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 5cc6b1921450..1cd8940314db 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -184,6 +184,7 @@ fn serve_args_forward_python_flags_with_separator() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Off, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1023,6 +1024,7 @@ fn frontend_args_accept_json() { engine_ready_timeout_secs: 600, tool_call_parser: None, reasoning_parser: None, + tool_strict_level: Off, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1693,6 +1695,7 @@ fn serve_args_accept_handshake_aliases() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Off, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1850,6 +1853,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { }, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Off, renderer: Auto, language_model_only: false, chat_template: None, @@ -1942,6 +1946,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { }, tool_call_parser: Auto, reasoning_parser: Auto, + tool_strict_level: Off, renderer: Auto, language_model_only: false, chat_template: None, @@ -2055,6 +2060,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present }, tool_call_parser: None, reasoning_parser: None, + tool_strict_level: Off, renderer: Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 4c337adbef8f..afcfaeccfb0a 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -14,6 +14,7 @@ use serde_json::Value; use vllm_chat::multimodal::MmLimitPerPrompt; use vllm_chat::{ ChatTemplateContentFormatOption, GenerationConfigMode, ParserSelection, RendererSelection, + ToolStrictLevel, }; use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode}; use vllm_text::backend::hf::HfOverrides; @@ -229,6 +230,8 @@ pub struct Config { pub tool_call_parser: ParserSelection, /// Reasoning parser selection. pub reasoning_parser: ParserSelection, + /// Server-side floor for tool-call structural tags. + pub tool_strict_level: ToolStrictLevel, /// Chat renderer selection. pub renderer: RendererSelection, /// Disable frontend-side multimodal preprocessing and render the model as diff --git a/rust/src/server/src/grpc/tests.rs b/rust/src/server/src/grpc/tests.rs index f4b35c9df4a8..18b6e6eb2990 100644 --- a/rust/src/server/src/grpc/tests.rs +++ b/rust/src/server/src/grpc/tests.rs @@ -244,6 +244,7 @@ impl ChatBackend for FakeTextBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } @@ -292,6 +293,7 @@ impl ChatBackend for FakeMultimodalBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 49d3d57f3264..91564db73c5b 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -139,7 +139,8 @@ async fn build_state(config: &Config) -> Result> { let chat = ChatLlm::new(text, chat_backend) .with_tool_call_parser(config.tool_call_parser.clone()) - .with_reasoning_parser(config.reasoning_parser.clone()); + .with_reasoning_parser(config.reasoning_parser.clone()) + .with_tool_strict_level(config.tool_strict_level); let state = Arc::new( AppState::new(served_model_names, chat) diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs index 4fc55fe60be9..343e98152cb7 100644 --- a/rust/src/server/src/render.rs +++ b/rust/src/server/src/render.rs @@ -10,7 +10,7 @@ use tokio_util::sync::CancellationToken; use tracing::info; use vllm_chat::{ ChatRequestProcessor, ChatTemplateContentFormatOption, LoadModelBackendsOptions, - ParserSelection, RendererSelection, load_model_backends, + ParserSelection, RendererSelection, ToolStrictLevel, load_model_backends, }; use vllm_text::TextRequestProcessor; use vllm_text::backend::hf::HfOverrides; @@ -32,6 +32,7 @@ pub struct RenderConfig { pub port: u16, pub tool_call_parser: ParserSelection, pub reasoning_parser: ParserSelection, + pub tool_strict_level: ToolStrictLevel, pub renderer: RendererSelection, pub chat_template: Option, pub default_chat_template_kwargs: HashMap, @@ -88,10 +89,12 @@ async fn build_state(config: &RenderConfig) -> Result> { let max_model_len = config.max_model_len.unwrap_or(u32::MAX); let text = TextRequestProcessor::new(loaded.text_backend, max_model_len) .with_max_logprobs(config.max_logprobs); - let chat = ChatRequestProcessor::render_only(loaded.chat_backend).with_parser_selections( - config.tool_call_parser.clone(), - config.reasoning_parser.clone(), - ); + let chat = ChatRequestProcessor::render_only(loaded.chat_backend) + .with_parser_selections( + config.tool_call_parser.clone(), + config.reasoning_parser.clone(), + ) + .with_tool_strict_level(config.tool_strict_level); Ok(Arc::new(RenderState { model: config.model.clone(), served_model_names, @@ -154,6 +157,7 @@ mod tests { shutdown.cancel(); let error = serve_render( RenderConfig { + tool_strict_level: ToolStrictLevel::Off, model: "test-model".to_string(), revision: None, hf_overrides: Default::default(), diff --git a/rust/src/server/src/routes/http_client_tests.rs b/rust/src/server/src/routes/http_client_tests.rs index a0e0872e64ce..762d974b9b6b 100644 --- a/rust/src/server/src/routes/http_client_tests.rs +++ b/rust/src/server/src/routes/http_client_tests.rs @@ -168,6 +168,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 4731958e2f87..8edcb3296070 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -517,6 +517,7 @@ impl ChatBackend for FakeChatBackend { self.tokenizer(), options.tool_call_parser, options.reasoning_parser, + options.tool_strict_level, )?)) } } diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 9e16bbab2d03..2a18e2be10f1 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -9,7 +9,6 @@ from xgrammar import Grammar, StructuralTag, builtin_structural_tag from xgrammar.testing import _is_grammar_accept_string -from vllm import envs from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedFunction, ChatCompletionNamedToolChoiceParam, @@ -39,6 +38,7 @@ get_function_parameters, get_model_structural_tag, ) +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel @pytest.fixture @@ -234,11 +234,11 @@ def test_get_model_structural_tag_supports_all_xgrammar_builtins( def test_get_model_structural_tag_supports_vllm_hermes( - sample_tools: list[ChatCompletionToolsParam], + sample_tools_strict: list[ChatCompletionToolsParam], ): tag = get_model_structural_tag( model="hermes", - tools=sample_tools, + tools=sample_tools_strict, tool_choice="required", reasoning=False, ) @@ -276,6 +276,22 @@ def test_get_model_structural_tag_supports_vllm_hermes( assert content["json_schema"] == expected_schema +def test_hermes_required_without_strict_leaves_arguments_free( + sample_tools: list[ChatCompletionToolsParam], +): + tag = get_model_structural_tag( + model="hermes", + tools=sample_tools, + tool_choice="required", + reasoning=False, + ) + + assert isinstance(tag, StructuralTag) + tags = tag.model_dump()["format"]["tags"] + assert tags + assert all(tag_dump["content"]["json_schema"] is True for tag_dump in tags) + + def test_hermes_required_tool_calls_use_empty_separator(): tools = [ ChatCompletionToolsParam( @@ -364,10 +380,19 @@ def _k3_tools(*calls: str) -> str: return f"{_K3_TOOLS_OPEN}{''.join(calls)}{_K3_TOOLS_CLOSE}" -def _k3_grammar(tool_choice, tools=None): +def _as_strict(tools: list[ChatCompletionToolsParam]) -> list[ChatCompletionToolsParam]: + for tool in tools: + tool.function.strict = True + return tools + + +def _k3_grammar(tool_choice, tools=None, *, strict: bool = True): + """Build the K3 grammar; tools are strict by default so the typed + argument channel is exercised (an unset ``strict`` leaves arguments free).""" + tools = tools if tools is not None else _k3_tools_by_name() tag = get_model_structural_tag( model="kimi_k3", - tools=tools if tools is not None else _k3_tools_by_name(), + tools=_as_strict(tools) if strict else tools, tool_choice=tool_choice, reasoning=False, ) @@ -470,6 +495,18 @@ def test_kimi_k3_required_rejects_invalid(body: str): assert not _is_grammar_accept_string(_k3_grammar("required"), body) +def test_kimi_k3_required_without_strict_leaves_arguments_free(): + # An unset ``strict`` pins the call envelope only: a value the schema + # would reject is accepted, and the declared-required argument may be + # omitted. + grammar = _k3_grammar("required", strict=False) + body = _k3_response() + _k3_tools( + _k3_call("get_weather", _k3_arg("days", "number", "abc")) + ) + assert _is_grammar_accept_string(grammar, body) + assert not _is_grammar_accept_string(_k3_grammar("required"), body) + + def test_kimi_k3_schema_without_required_accepts_empty_call(): tools = [ ChatCompletionToolsParam( @@ -838,11 +875,14 @@ def _pins_argument_schema(tag: StructuralTag) -> bool: return '"json_schema": {' in json.dumps(tag.model_dump(), ensure_ascii=False) +def _dumped(tag: StructuralTag) -> str: + return json.dumps(tag.model_dump(), ensure_ascii=False) + + def test_tool_strict_level_off_is_the_default( sample_tools: list[ChatCompletionToolsParam], ): - """auto + no strict tool still gets no tag unless the operator opts in.""" - assert envs.VLLM_TOOL_STRICT_LEVEL == "off" + """Auto + no strict tool gets no tag unless the operator raises the floor.""" assert ( get_model_structural_tag( model="deepseek_v4", @@ -858,14 +898,13 @@ def test_tool_strict_level_off_is_the_default( def test_tool_strict_level_function_lifts_auto_gate( model: str, sample_tools: list[ChatCompletionToolsParam], - monkeypatch: pytest.MonkeyPatch, ): - monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "function", raising=False) tag = get_model_structural_tag( model=model, tools=sample_tools, tool_choice="auto", reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, ) assert tag is not None @@ -875,17 +914,14 @@ def test_tool_strict_level_function_lifts_auto_gate( def test_tool_strict_level_function_pins_envelope_only( model: str, sample_tools: list[ChatCompletionToolsParam], - monkeypatch: pytest.MonkeyPatch, ): - """xgrammar constrains arguments for tools whose ``strict`` is unset, so - the level must mark them non-strict explicitly or it would behave like - "parameter". The request's own tools stay untouched.""" - monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "function", raising=False) + """The request's own tools stay untouched; unset ``strict`` stays free.""" tag = get_model_structural_tag( model=model, tools=sample_tools, tool_choice="auto", reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, ) assert tag is not None @@ -897,14 +933,13 @@ def test_tool_strict_level_function_pins_envelope_only( def test_tool_strict_level_parameter_pins_argument_schemas( model: str, sample_tools: list[ChatCompletionToolsParam], - monkeypatch: pytest.MonkeyPatch, ): - monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "parameter", raising=False) tag = get_model_structural_tag( model=model, tools=sample_tools, tool_choice="auto", reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, ) assert tag is not None @@ -913,20 +948,59 @@ def test_tool_strict_level_parameter_pins_argument_schemas( def test_tool_strict_level_function_keeps_client_strict_tools_strict( sample_tools_strict: list[ChatCompletionToolsParam], - monkeypatch: pytest.MonkeyPatch, ): - monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "function", raising=False) tag = get_model_structural_tag( model="deepseek_v4", tools=sample_tools_strict, tool_choice="auto", reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, ) assert tag is not None assert _pins_argument_schema(tag) +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_absent_strict_is_not_pinned_next_to_a_strict_tool(model: str): + """A strict tool must not drag a neighbour with unset ``strict`` into + schema enforcement.""" + weather = ChatCompletionToolsParam( + type="function", + function={ + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + ) + search = ChatCompletionToolsParam( + type="function", + function={ + "name": "search", + "parameters": { + "type": "object", + "properties": {"query": {"type": "string"}}, + "required": ["query"], + }, + }, + ) + tag = get_model_structural_tag( + model=model, + tools=[weather, search], + tool_choice="auto", + reasoning=False, + ) + + assert tag is not None + dumped = _dumped(tag) + assert '"city"' in dumped + assert '"query"' not in dumped + + @pytest.mark.parametrize( "tool_choice", [ @@ -936,28 +1010,42 @@ def test_tool_strict_level_function_keeps_client_strict_tools_strict( ), ], ) -def test_tool_strict_level_function_never_relaxes_forced_tool_choice( +@pytest.mark.parametrize("model", sorted(XGRAMMAR_BUILTIN_STRUCTURAL_TAG_MODELS)) +def test_forced_tool_choice_pins_schema_only_for_strict_tools( + model: str, tool_choice: ToolChoice, sample_tools: list[ChatCompletionToolsParam], - monkeypatch: pytest.MonkeyPatch, + sample_tools_strict: list[ChatCompletionToolsParam], ): - """The level is a floor: required / named calls keep their schema pinning.""" - monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "function", raising=False) - tag = get_model_structural_tag( - model="deepseek_v4", + """Required / named always constrain the call; the argument schema + follows the per-tool ``strict`` (or the parameter level).""" + free = get_model_structural_tag( + model=model, + tools=sample_tools, + tool_choice=tool_choice, + reasoning=False, + strict_level=ToolStrictLevel.FUNCTION, + ) + pinned = get_model_structural_tag( + model=model, + tools=sample_tools_strict, + tool_choice=tool_choice, + reasoning=False, + ) + floored = get_model_structural_tag( + model=model, tools=sample_tools, tool_choice=tool_choice, reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, ) - assert tag is not None - assert _pins_argument_schema(tag) + assert free is not None and not _pins_argument_schema(free) + assert pinned is not None and _pins_argument_schema(pinned) + assert floored is not None and _pins_argument_schema(floored) -def test_tool_strict_level_parameter_overrides_client_strict_false( - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "parameter", raising=False) +def test_tool_strict_level_parameter_overrides_client_strict_false(): tools = [ ChatCompletionToolsParam( type="function", @@ -973,23 +1061,14 @@ def test_tool_strict_level_parameter_overrides_client_strict_false( tools=tools, tool_choice="required", reasoning=False, + strict_level=ToolStrictLevel.PARAMETER, ) assert tag is not None assert _pins_argument_schema(tag) -def test_tool_strict_level_unknown_value_falls_back_to_off( - sample_tools: list[ChatCompletionToolsParam], - monkeypatch: pytest.MonkeyPatch, -): - monkeypatch.setattr(envs, "VLLM_TOOL_STRICT_LEVEL", "nonsense", raising=False) - assert ( - get_model_structural_tag( - model="deepseek_v4", - tools=sample_tools, - tool_choice="auto", - reasoning=False, - ) - is None - ) +def test_tool_strict_level_from_name(): + assert ToolStrictLevel.from_name("Parameter") is ToolStrictLevel.PARAMETER + with pytest.raises(ValueError, match="expected one of off, function, parameter"): + ToolStrictLevel.from_name("strict") diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index e90d70e289a4..d774f1d91e9f 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -112,6 +112,7 @@ def __init__( reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "off", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, @@ -127,6 +128,7 @@ def __init__( return_tokens_as_token_ids=return_tokens_as_token_ids, reasoning_parser=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, tool_parser=tool_parser, enable_prompt_tokens_details=enable_prompt_tokens_details, enable_force_include_usage=enable_force_include_usage, diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index 788558645243..8a80141e788b 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -249,6 +249,7 @@ def __init__( reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "off", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, @@ -265,6 +266,7 @@ def __init__( return_tokens_as_token_ids=return_tokens_as_token_ids, reasoning_parser=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, tool_parser=tool_parser, enable_prompt_tokens_details=enable_prompt_tokens_details, enable_force_include_usage=enable_force_include_usage, diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index ac995a4f031a..189c4e6779b0 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -112,6 +112,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, tool_server=state.tool_server, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, @@ -137,6 +138,7 @@ async def init_generate_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, @@ -178,6 +180,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, @@ -198,6 +201,7 @@ async def init_generate_state( return_tokens_as_token_ids=args.return_tokens_as_token_ids, enable_auto_tools=args.enable_auto_tool_choice, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, enable_prompt_tokens_details=args.enable_prompt_tokens_details, enable_force_include_usage=args.enable_force_include_usage, diff --git a/vllm/entrypoints/launchers/api_server/app_state.py b/vllm/entrypoints/launchers/api_server/app_state.py index 09bb0ec63020..d33be659494c 100644 --- a/vllm/entrypoints/launchers/api_server/app_state.py +++ b/vllm/entrypoints/launchers/api_server/app_state.py @@ -95,6 +95,7 @@ async def init_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, @@ -111,6 +112,7 @@ async def init_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.structured_outputs_config.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 71ae94c10635..80b545301996 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -20,6 +20,7 @@ ) from vllm.entrypoints.openai.models.protocol import LoRAModulePath from vllm.tool_parsers import ToolParserManager +from vllm.tool_parsers.tool_strict_level import ToolStrictLevelName from vllm.utils.argparse_utils import FlexibleArgumentParser from .utils.constants import ( @@ -116,6 +117,13 @@ class BaseFrontendArgs: """Special the tool parser plugin write to parse the model-generated tool into OpenAI API format, the name register in this plugin can be used in `--tool-call-parser`.""" + tool_strict_level: ToolStrictLevelName = "off" + """Server-side floor for structural-tag based tool calling, applied on top + of the per-tool `strict` field. `off` constrains a `tool_choice="auto"` + request only when a tool sets `strict: true`; `function` constrains the + tool-call envelope (markup and function name) for every request with + tools; `parameter` additionally pins argument schemas, as if every tool + were `strict: true`.""" tool_server: str | None = None """Comma-separated list of host:port pairs (IPv4, IPv6, or hostname). Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for diff --git a/vllm/entrypoints/launchers/render/app_state.py b/vllm/entrypoints/launchers/render/app_state.py index 33988443d1d9..1d242977be29 100644 --- a/vllm/entrypoints/launchers/render/app_state.py +++ b/vllm/entrypoints/launchers/render/app_state.py @@ -60,6 +60,7 @@ async def init_render_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, @@ -76,6 +77,7 @@ async def init_render_app_state( enable_auto_tools=args.enable_auto_tool_choice, exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none, tool_parser=args.tool_call_parser, + tool_strict_level=args.tool_strict_level, reasoning_parser=args.reasoning_parser, default_chat_template_kwargs=default_chat_template_kwargs, log_error_stack=args.log_error_stack, diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index 51c0af8e8f15..ac82ae130d3c 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -132,6 +132,7 @@ def __init__( enable_auto_tools: bool = False, exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "off", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, @@ -161,6 +162,7 @@ def __init__( tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index ee4f7d55ca0d..9dead9a4f59b 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -109,6 +109,7 @@ def __init__( reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, + tool_strict_level: str = "off", tool_server: ToolServer | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, @@ -135,6 +136,7 @@ def __init__( tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=self.model_config.model, is_harmony=self.model_config.hf_config.model_type == "gpt_oss", ) diff --git a/vllm/envs.py b/vllm/envs.py index 123a72ebf1a9..c512df457adc 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -239,7 +239,6 @@ VLLM_MAX_TOKENS_PER_EXPERT_FP4_MOE: int = 163840 VLLM_TOOL_PARSE_REGEX_TIMEOUT_SECONDS: int = 1 VLLM_ENFORCE_STRICT_TOOL_CALLING: bool = True - VLLM_TOOL_STRICT_LEVEL: str = "off" VLLM_MQ_MAX_CHUNK_BYTES_MB: int = 16 VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS: int = 300 VLLM_WORKER_SHUTDOWN_TIMEOUT_SECONDS: int = 5 @@ -1760,12 +1759,6 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_ENFORCE_STRICT_TOOL_CALLING": lambda: ( os.getenv("VLLM_ENFORCE_STRICT_TOOL_CALLING", "True").lower() in ("true", "1") ), - # Server-side floor for structural-tag based tool calling, applied on top - # of the per-tool `strict` field: "off" (default) only constrains an - # "auto" request when a tool sets strict, "function" constrains the - # tool-call envelope for every request with tools, "parameter" also pins - # argument schemas. Mirrors SGLANG_TOOL_STRICT_LEVEL. - "VLLM_TOOL_STRICT_LEVEL": lambda: os.getenv("VLLM_TOOL_STRICT_LEVEL", "off"), # Control the max chunk bytes (in MB) for the rpc message queue. # Object larger than this threshold will be broadcast to worker # processes via zmq. diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index af448e0951c6..16ba6c8427c7 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -36,6 +36,7 @@ extract_named_tool_call_streaming, extract_required_tool_call_streaming, ) +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel logger = init_logger(__name__) @@ -95,6 +96,8 @@ class Parser: # Subclasses should override these if they use specific parser classes reasoning_parser_cls: type[ReasoningParser] | None = None tool_parser_cls: type[ToolParser] | None = None + # Server-side floor for tool-call structural tags (--tool-strict-level). + tool_strict_level: ToolStrictLevel = ToolStrictLevel.OFF def __init__( self, @@ -415,6 +418,7 @@ def _apply_structural_tag( structure_tag = self._tool_parser.get_structural_tag( request, reasoning=False, + strict_level=self.tool_strict_level, ) if structure_tag is None: return request diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index a8efd3f0eaa1..e8aed9fea736 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -6,6 +6,7 @@ from typing import TYPE_CHECKING from vllm.logger import init_logger +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel if TYPE_CHECKING: from vllm.parser.abstract_parser import Parser @@ -77,6 +78,7 @@ def get_parser( enable_auto_tools: bool = False, model_name: str | None = None, is_harmony: bool = False, + tool_strict_level: str = "off", ) -> type[Parser] | None: """Get a Parser that handles both reasoning and tool parsing. @@ -89,6 +91,8 @@ def get_parser( model_name: The model name for parser-specific warnings. is_harmony: Whether the selected model uses the Harmony format. If True, HarmonyParser is always returned. + tool_strict_level: Server-side floor for tool-call structural + tags (``--tool-strict-level``). Returns: A Parser class, or None if neither parser is specified. @@ -105,11 +109,14 @@ def get_parser( if reasoning_parser_cls is None and tool_parser_cls is None: return None + strict_level = ToolStrictLevel.from_name(tool_strict_level) + if is_harmony: from vllm.parser.harmony import HarmonyParser HarmonyParser.reasoning_parser_cls = reasoning_parser_cls HarmonyParser.tool_parser_cls = tool_parser_cls + HarmonyParser.tool_strict_level = strict_level return HarmonyParser if reasoning_parser_name == "kimi_k3" or tool_parser_name == "kimi_k3": @@ -121,6 +128,7 @@ def get_parser( class _KimiK3Parser(KimiK3Parser): reasoning_parser_cls = r_cls tool_parser_cls = t_cls + tool_strict_level = strict_level return _KimiK3Parser @@ -136,6 +144,7 @@ class _KimiK3Parser(KimiK3Parser): class _CohereCommandParser(CohereCommandParser): reasoning_parser_cls = r_cls tool_parser_cls = t_cls + tool_strict_level = strict_level return _CohereCommandParser @@ -147,5 +156,6 @@ class _CohereCommandParser(CohereCommandParser): class _Parser(DelegatingParser): reasoning_parser_cls = r_cls tool_parser_cls = t_cls + tool_strict_level = strict_level return _Parser diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index 5f30b552ba83..b611fad9b627 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -54,6 +54,7 @@ def __init__( exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, reasoning_parser: str | None = None, + tool_strict_level: str = "off", default_chat_template_kwargs: dict[str, Any] | None = None, log_error_stack: bool = False, ) -> None: @@ -68,6 +69,7 @@ def __init__( tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=model_config.model, is_harmony=self.use_harmony, ) diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index 2ab5725b73fc..db4de6799b86 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -127,6 +127,7 @@ def __init__( exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, reasoning_parser: str | None = None, + tool_strict_level: str = "off", default_chat_template_kwargs: dict[str, Any] | None = None, log_error_stack: bool = False, ) -> None: @@ -141,6 +142,7 @@ def __init__( tool_parser_name=tool_parser, reasoning_parser_name=reasoning_parser, enable_auto_tools=enable_auto_tools, + tool_strict_level=tool_strict_level, model_name=model_config.model, is_harmony=self.use_harmony, ) diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index 3847afa9b227..5e597d1195bb 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -30,6 +30,7 @@ StructuredOutputsParams, ) from vllm.tokenizers import TokenizerLike +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel from vllm.tool_parsers.utils import Tool, get_json_schema_from_tools from vllm.utils.collection_utils import is_list_of from vllm.utils.import_utils import import_plugin @@ -168,6 +169,7 @@ def get_structural_tag( request: ChatCompletionRequest | ResponsesRequest, *, reasoning: bool = False, + strict_level: ToolStrictLevel = ToolStrictLevel.OFF, ): if self.structural_tag_model is None: return None @@ -180,6 +182,7 @@ def get_structural_tag( tools=request.tools, tool_choice=request.tool_choice, reasoning=reasoning, + strict_level=strict_level, ) def extract_tool_calls( diff --git a/vllm/tool_parsers/hy_v4_tool_parser.py b/vllm/tool_parsers/hy_v4_tool_parser.py index c2f82fc4a7be..f73ade9e27ae 100644 --- a/vllm/tool_parsers/hy_v4_tool_parser.py +++ b/vllm/tool_parsers/hy_v4_tool_parser.py @@ -25,6 +25,7 @@ from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike from vllm.tool_parsers.abstract_tool_parser import ToolParser +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel from vllm.tool_parsers.utils import partial_tag_overlap if TYPE_CHECKING: @@ -994,6 +995,7 @@ def get_structural_tag( request: ChatCompletionRequest | ResponsesRequest, *, reasoning: bool = False, + strict_level: ToolStrictLevel = ToolStrictLevel.OFF, ) -> StructuralTag | None: """Build a structural tag matching HYV4's tool tokens. @@ -1009,6 +1011,7 @@ def get_structural_tag( Args: request: The request being adjusted. reasoning: Whether the grammar also covers the reasoning phase. + strict_level: Server-side floor from ``--tool-strict-level``. Returns: The structural tag, or None when structural tagging does not apply. @@ -1041,6 +1044,7 @@ def get_structural_tag( tool_choice=request.tool_choice, reasoning=reasoning, token_suffix=self._extractor.token_suffix, + strict_level=strict_level, ) except Exception: logger.warning( diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 81d8752f99ab..6f07f250a5ab 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import enum from collections.abc import Callable, Sequence from typing import Any, Literal, TypeAlias @@ -32,14 +31,11 @@ TriggeredTagsFormat, ) -from vllm import envs from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionNamedToolChoiceParam, ChatCompletionToolsParam, ) -from vllm.logger import init_logger - -logger = init_logger(__name__) +from vllm.tool_parsers.tool_strict_level import ToolStrictLevel ToolChoice: TypeAlias = ( Literal["none", "auto", "required"] @@ -99,34 +95,6 @@ def decorator(func: StructuralTagBuilder) -> StructuralTagBuilder: return decorator -class ToolStrictLevel(enum.IntEnum): - """Server-side floor for tool-call structural tags (VLLM_TOOL_STRICT_LEVEL). - - OFF: only tools the client marked ``strict`` constrain an "auto" - request (default). - FUNCTION: constrain the tool-call envelope for every request with tools. - PARAMETER: additionally pin argument schemas, as if every tool were - ``strict``. - """ - - OFF = 0 - FUNCTION = 1 - PARAMETER = 2 - - -def _tool_strict_level() -> ToolStrictLevel: - raw = envs.VLLM_TOOL_STRICT_LEVEL.strip().lower() - try: - return ToolStrictLevel[raw.upper()] - except KeyError: - logger.warning_once( - "Unknown VLLM_TOOL_STRICT_LEVEL %r; expected one of %s. Using 'off'.", - raw, - ", ".join(level.name.lower() for level in ToolStrictLevel), - ) - return ToolStrictLevel.OFF - - def _tool_is_strict(tool: ChatCompletionToolsParam | ResponsesTool) -> bool: if isinstance(tool, FunctionTool): return tool.strict is True @@ -155,27 +123,29 @@ def _with_tool_strict( return tool -def _apply_tool_strict_level( +def _resolve_tool_strictness( tools: Sequence[ChatCompletionToolsParam | ResponsesTool], tool_choice: ToolChoice, + strict_level: ToolStrictLevel, ) -> Sequence[ChatCompletionToolsParam | ResponsesTool] | None: - """Apply VLLM_TOOL_STRICT_LEVEL; ``None`` means no structural tag.""" - level = _tool_strict_level() - - if tool_choice == "auto" and not _any_tool_strict(tools): - if level < ToolStrictLevel.FUNCTION: - return None - if level == ToolStrictLevel.FUNCTION: - # xgrammar pins argument schemas for tools whose ``strict`` is - # unset, so mark them non-strict to constrain the envelope only. - return [_with_tool_strict(tool, False) for tool in tools] - - if level >= ToolStrictLevel.PARAMETER: - return [ - tool if _tool_is_strict(tool) else _with_tool_strict(tool, True) - for tool in tools - ] - return tools + """Decide whether a structural tag applies and pin each tool's ``strict``. + + A tool without an explicit ``strict`` is treated as non-strict: its call + envelope is still constrained, but its arguments stay free unless the + server level is PARAMETER. ``None`` means no structural tag. + """ + if ( + tool_choice == "auto" + and strict_level == ToolStrictLevel.OFF + and not _any_tool_strict(tools) + ): + return None + return [ + _with_tool_strict( + tool, strict_level >= ToolStrictLevel.PARAMETER or _tool_is_strict(tool) + ) + for tool in tools + ] def get_model_structural_tag( @@ -184,12 +154,13 @@ def get_model_structural_tag( tool_choice: ToolChoice, reasoning: bool, token_suffix: str = "", + strict_level: ToolStrictLevel = ToolStrictLevel.OFF, ) -> StructuralTag | None: """Build a structural tag with xgrammar's builtin model templates.""" if not tools or tool_choice == "none": return None - tools = _apply_tool_strict_level(tools, tool_choice) + tools = _resolve_tool_strictness(tools, tool_choice, strict_level) if tools is None: return None diff --git a/vllm/tool_parsers/tool_strict_level.py b/vllm/tool_parsers/tool_strict_level.py new file mode 100644 index 000000000000..1485ba5c712f --- /dev/null +++ b/vllm/tool_parsers/tool_strict_level.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import enum +from typing import Literal + +ToolStrictLevelName = Literal["off", "function", "parameter"] + + +class ToolStrictLevel(enum.IntEnum): + """Server-side floor for tool-call structural tags (``--tool-strict-level``). + + OFF: constrain a ``tool_choice="auto"`` request only when a tool sets + ``strict: true``. + FUNCTION: constrain the tool-call envelope for every request with tools. + PARAMETER: additionally pin argument schemas, as if every tool were + ``strict: true``. + """ + + OFF = 0 + FUNCTION = 1 + PARAMETER = 2 + + @classmethod + def from_name(cls, name: str) -> "ToolStrictLevel": + try: + return cls[name.strip().upper()] + except KeyError: + expected = ", ".join(level.name.lower() for level in cls) + raise ValueError( + f"Unknown tool strict level {name!r}; expected one of {expected}." + ) from None From c5479631b94b85b631f0532209ee5f951cfbfb54 Mon Sep 17 00:00:00 2001 From: lazymio Date: Thu, 17 Sep 2026 13:00:55 +0000 Subject: [PATCH 3/5] no sglang stuff --- docs/features/tool_calling.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 2ae356a0c6ba..436d0de80758 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -136,7 +136,7 @@ VLLM_ENFORCE_STRICT_TOOL_CALLING=false vllm serve ... ### Server-Side Strictness Floor -Most OpenAI-compatible clients and agent frameworks never set `strict` on their tools, so with `tool_choice="auto"` the model generates tool calls without any grammar and malformed markup can leak into the response. The `--tool-strict-level` option lets the server operator raise the floor for every request that carries tools, independently of what the client declares (similar to SGLang's `SGLANG_TOOL_STRICT_LEVEL`): +Most OpenAI-compatible clients and agent frameworks never set `strict` on their tools, so with `tool_choice="auto"` the model generates tool calls without any grammar and malformed markup can leak into the response. The `--tool-strict-level` option lets the server operator raise the floor for every request that carries tools, independently of what the client declares: | Value | Behavior | | --- | --- | From 583d60efe014608d6ad86452a96f17c368b7ab42 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 18 Sep 2026 03:06:32 +0000 Subject: [PATCH 4/5] [Frontend] Fix tool strictness docs and Rust example Assisted-by: Codex Signed-off-by: Bugen Zhao --- docs/features/tool_calling.md | 20 +++++++++++-------- .../examples/external_engine_openai_qwen.rs | 2 ++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 436d0de80758..545cc4f4db1e 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -109,14 +109,18 @@ vLLM supports the `tool_choice='none'` option in the chat completion API. When t ## Constrained Decoding Behavior -Whether vLLM enforces the tool parameter schema during generation depends on the `tool_choice` mode, the per-tool `strict` field, and the server-side `--tool-strict-level`: +Structural-tag parsers resolve call obligation, grammar activation, and argument-schema enforcement separately. With structural-tag enforcement enabled: -| `tool_choice` value | Schema-constrained decoding | Behavior | +| `tool_choice` value | Call obligation | Structural-tag activation | | --- | --- | --- | -| Named function | Yes (via structured outputs backend) | Arguments are guaranteed to be valid JSON conforming to the function's parameter schema. With a structural-tag parser, the call is always constrained but the argument schema is pinned only for tools with `strict: true` (or `--tool-strict-level parameter`). | -| `"required"` | Yes (via structured outputs backend) | Same as named function. The model must produce at least one tool call. | -| `"auto"` | Only when `strict: true` is set on at least one tool, or the server raises the floor via `--tool-strict-level` | Structural-tag parsers constrain the call envelope, and pin the argument schema of each tool that opts in with `strict: true`. Without either, the model generates freely and tool calls are extracted from raw text. | -| `"none"` | N/A | No tool calls are produced. | +| Named function | Call the selected function | Always | +| `"required"` | Produce at least one tool call | Always | +| `"auto"` | Tool calls are optional | When at least one tool sets `strict: true`, or `--tool-strict-level` is `function` or `parameter` | +| `"none"` | Tool calling is disabled | Disabled | + +When a structural tag applies, each tool's declared parameter schema is enforced only when that tool sets `strict: true` or the server uses `--tool-strict-level parameter`. Tools with omitted or false `strict` receive broad argument-syntax constraints at levels `off` and `function`, including for required and named calls. + +For parsers using schema-derived JSON constraints, required and named calls continue to enforce the declared parameter schemas. ### Strict Mode @@ -178,8 +182,8 @@ All Nous Research Hermes-series models newer than Hermes 2 Pro should be support * `NousResearch/Hermes-2-Theta-*` * `NousResearch/Hermes-3-*` -_Note that the Hermes 2 **Theta** models are known to have degraded tool call quality and capabilities due to the merge -step in their creation_. +*Note that the Hermes 2 **Theta** models are known to have degraded tool call quality and capabilities due to the merge +step in their creation*. Flags: `--tool-call-parser hermes` diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index ad3872843a1f..5177658cefa6 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -16,6 +16,7 @@ use clap::Parser; use futures::StreamExt as _; use tokio_util::sync::CancellationToken; use tracing_subscriber::EnvFilter; +use vllm_chat::ToolStrictLevel; use vllm_engine_core_client::TransportMode; use vllm_server::{ ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig, @@ -70,6 +71,7 @@ async fn main() -> Result<()> { }, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, + tool_strict_level: ToolStrictLevel::Off, renderer: RendererSelection::Auto, language_model_only: false, chat_template: None, From c4711ee8c711d6741cf384ba8dd69ef03d1bbb0e Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Fri, 18 Sep 2026 03:16:46 +0000 Subject: [PATCH 5/5] [Frontend] Rename default tool strictness level to auto Assisted-by: Codex Signed-off-by: Bugen Zhao --- docs/features/tool_calling.md | 4 +- rust/src/chat/src/backend/hf.rs | 2 +- rust/src/chat/src/lib.rs | 4 +- rust/src/chat/src/output/default/mod.rs | 8 ++-- .../chat/src/output/default/structural_tag.rs | 44 ++++++++++++------- rust/src/chat/src/parser/mod.rs | 14 +++--- rust/src/chat/tests/roundtrip.rs | 2 +- rust/src/cmd/src/cli.rs | 6 +-- rust/src/cmd/src/cli/tests.rs | 12 ++--- .../examples/external_engine_openai_qwen.rs | 2 +- rust/src/server/src/render.rs | 2 +- .../test_structural_tag_registry.py | 7 ++- vllm/entrypoints/anthropic/serving.py | 2 +- vllm/entrypoints/cohere/serving.py | 2 +- vllm/entrypoints/launchers/cli_args.py | 6 +-- .../openai/chat_completion/serving.py | 2 +- vllm/entrypoints/openai/responses/serving.py | 2 +- vllm/parser/abstract_parser.py | 2 +- vllm/parser/parser_manager.py | 2 +- vllm/renderers/online_derenderer.py | 2 +- vllm/renderers/online_renderer.py | 2 +- vllm/tool_parsers/abstract_tool_parser.py | 2 +- vllm/tool_parsers/hy_v4_tool_parser.py | 2 +- vllm/tool_parsers/structural_tag_registry.py | 4 +- vllm/tool_parsers/tool_strict_level.py | 8 ++-- 25 files changed, 79 insertions(+), 66 deletions(-) diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 545cc4f4db1e..fb70c5024835 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -118,7 +118,7 @@ Structural-tag parsers resolve call obligation, grammar activation, and argument | `"auto"` | Tool calls are optional | When at least one tool sets `strict: true`, or `--tool-strict-level` is `function` or `parameter` | | `"none"` | Tool calling is disabled | Disabled | -When a structural tag applies, each tool's declared parameter schema is enforced only when that tool sets `strict: true` or the server uses `--tool-strict-level parameter`. Tools with omitted or false `strict` receive broad argument-syntax constraints at levels `off` and `function`, including for required and named calls. +When a structural tag applies, each tool's declared parameter schema is enforced only when that tool sets `strict: true` or the server uses `--tool-strict-level parameter`. Tools with omitted or false `strict` receive broad argument-syntax constraints at levels `auto` and `function`, including for required and named calls. For parsers using schema-derived JSON constraints, required and named calls continue to enforce the declared parameter schemas. @@ -144,7 +144,7 @@ Most OpenAI-compatible clients and agent frameworks never set `strict` on their | Value | Behavior | | --- | --- | -| `off` (default) | With `tool_choice="auto"`, structural tags apply only when a tool sets `strict: true`. | +| `auto` (default) | Follow the request's tool choice and per-tool strictness. Required/named choices activate structural tags; `tool_choice="auto"` activates them when at least one tool sets `strict: true`. | | `function` | Constrain the tool-call envelope (markup and the function name) for every request with tools, leaving argument contents free unless the client marked the tool `strict: true`. | | `parameter` | Additionally pin argument schemas for every tool, as if every tool had `strict: true`. | diff --git a/rust/src/chat/src/backend/hf.rs b/rust/src/chat/src/backend/hf.rs index d16f37a8911b..07f0b2dce815 100644 --- a/rust/src/chat/src/backend/hf.rs +++ b/rust/src/chat/src/backend/hf.rs @@ -344,7 +344,7 @@ mod tests { let error = match backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { - tool_strict_level: crate::ToolStrictLevel::Off, + tool_strict_level: crate::ToolStrictLevel::Auto, tool_call_parser: &ParserSelection::Explicit("json".to_string()), reasoning_parser: &ParserSelection::Auto, }, diff --git a/rust/src/chat/src/lib.rs b/rust/src/chat/src/lib.rs index 42bb5633d027..0273396ebd85 100644 --- a/rust/src/chat/src/lib.rs +++ b/rust/src/chat/src/lib.rs @@ -87,7 +87,7 @@ impl ChatRequestProcessor { model_dtype: Some(model_dtype), tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, - tool_strict_level: ToolStrictLevel::Off, + tool_strict_level: ToolStrictLevel::Auto, } } @@ -98,7 +98,7 @@ impl ChatRequestProcessor { model_dtype: None, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, - tool_strict_level: ToolStrictLevel::Off, + tool_strict_level: ToolStrictLevel::Auto, } } diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index 0019d431941b..9fb3d02b114f 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -218,7 +218,7 @@ mod tests { tokenizer(), &selection, &selection, - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -233,7 +233,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Auto, - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -251,7 +251,7 @@ mod tests { tokenizer(), tool, reasoning, - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .unwrap(); } @@ -266,7 +266,7 @@ mod tests { tokenizer(), &ParserSelection::Auto, &ParserSelection::Explicit("gemma4".to_string()), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) { Ok(_) => panic!("expected mixed Gemma4 parser selection to fail"), Err(error) => error, diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs index c4eb285600b4..116cd53a4abe 100644 --- a/rust/src/chat/src/output/default/structural_tag.rs +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -160,7 +160,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag should build"); @@ -188,7 +188,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag should build"); @@ -219,7 +219,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag should build"); @@ -236,7 +236,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag decision should succeed"); @@ -255,7 +255,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag should build"); @@ -274,7 +274,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag should build"); @@ -295,7 +295,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag should build"); @@ -319,7 +319,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag should build"); @@ -336,7 +336,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag decision should succeed"); @@ -355,7 +355,7 @@ mod tests { apply_structural_tag_constraint( &mut request, parser.structural_tag_builder(), - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ) .expect("structural tag decision should succeed"); @@ -394,17 +394,17 @@ mod tests { let unset = build( ChatToolChoice::Required, vec![chat_tool("search", None)], - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ); let non_strict = build( ChatToolChoice::Required, vec![chat_tool("search", Some(false))], - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ); let strict = build( ChatToolChoice::Required, vec![chat_tool("search", Some(true))], - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ); assert_eq!(tag_string(&unset), tag_string(&non_strict)); @@ -416,7 +416,7 @@ mod tests { let mixed = build( ChatToolChoice::Auto, vec![chat_tool("weather", Some(true)), chat_tool("search", None)], - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ); let explicit = build( ChatToolChoice::Auto, @@ -424,7 +424,7 @@ mod tests { chat_tool("weather", Some(true)), chat_tool("search", Some(false)), ], - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ); assert_eq!(tag_string(&mixed), tag_string(&explicit)); @@ -440,7 +440,7 @@ mod tests { let strict = build( ChatToolChoice::Auto, vec![chat_tool("search", Some(true))], - ToolStrictLevel::Off, + ToolStrictLevel::Auto, ); let envelope_only = build( ChatToolChoice::Auto, @@ -454,6 +454,7 @@ mod tests { #[test] fn tool_strict_level_parses_case_insensitively() { + assert_eq!("AUTO".parse::(), Ok(ToolStrictLevel::Auto)); assert_eq!( "PARAMETER".parse::(), Ok(ToolStrictLevel::Parameter) @@ -462,7 +463,16 @@ mod tests { "function".parse::(), Ok(ToolStrictLevel::Function) ); - assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Off); + assert_eq!(ToolStrictLevel::default(), ToolStrictLevel::Auto); + assert_eq!( + serde_json::to_value(ToolStrictLevel::Auto).unwrap(), + json!("auto") + ); + assert_eq!( + serde_json::from_value::(json!("auto")).unwrap(), + ToolStrictLevel::Auto + ); + assert!("off".parse::().is_err()); assert!("strict".parse::().is_err()); } } diff --git a/rust/src/chat/src/parser/mod.rs b/rust/src/chat/src/parser/mod.rs index c42835b71e05..5a86b24934be 100644 --- a/rust/src/chat/src/parser/mod.rs +++ b/rust/src/chat/src/parser/mod.rs @@ -71,9 +71,9 @@ impl ParserSelection { SerializeDisplay, )] pub enum ToolStrictLevel { - /// Constrain a `tool_choice = "auto"` request only when a tool sets `strict: true`. + /// Derive tool constraints from the request's tool choice and per-tool strictness. #[default] - Off, + Auto, /// Constrain the tool-call envelope for every request with tools. Function, /// Additionally pin argument schemas, as if every tool were `strict: true`. @@ -81,7 +81,7 @@ pub enum ToolStrictLevel { } impl ToolStrictLevel { - pub const OFF_LITERAL: &str = "off"; + pub const AUTO_LITERAL: &str = "auto"; pub const FUNCTION_LITERAL: &str = "function"; pub const PARAMETER_LITERAL: &str = "parameter"; } @@ -90,8 +90,8 @@ impl FromStr for ToolStrictLevel { type Err = String; fn from_str(value: &str) -> Result { - if value.eq_ignore_ascii_case(Self::OFF_LITERAL) { - Ok(Self::Off) + if value.eq_ignore_ascii_case(Self::AUTO_LITERAL) { + Ok(Self::Auto) } else if value.eq_ignore_ascii_case(Self::FUNCTION_LITERAL) { Ok(Self::Function) } else if value.eq_ignore_ascii_case(Self::PARAMETER_LITERAL) { @@ -99,7 +99,7 @@ impl FromStr for ToolStrictLevel { } else { Err(format!( "unknown tool strict level {value:?}; expected one of {}, {}, {}", - Self::OFF_LITERAL, + Self::AUTO_LITERAL, Self::FUNCTION_LITERAL, Self::PARAMETER_LITERAL )) @@ -110,7 +110,7 @@ impl FromStr for ToolStrictLevel { impl fmt::Display for ToolStrictLevel { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(match self { - Self::Off => Self::OFF_LITERAL, + Self::Auto => Self::AUTO_LITERAL, Self::Function => Self::FUNCTION_LITERAL, Self::Parameter => Self::PARAMETER_LITERAL, }) diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 4557f5fdd0da..db3559c44f4c 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -717,7 +717,7 @@ async fn parse_completion( let processor = backends.chat_backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { - tool_strict_level: vllm_chat::ToolStrictLevel::Off, + tool_strict_level: vllm_chat::ToolStrictLevel::Auto, tool_call_parser: &case.tool_call_parser, reasoning_parser: &case.reasoning_parser, }, diff --git a/rust/src/cmd/src/cli.rs b/rust/src/cmd/src/cli.rs index 7b999b7047c1..5bdf66da57f8 100644 --- a/rust/src/cmd/src/cli.rs +++ b/rust/src/cmd/src/cli.rs @@ -135,7 +135,7 @@ pub struct RenderArgs { /// `none` to disable parsing. #[arg(long, default_value_t)] reasoning_parser: ParserSelection, - /// Server-side floor for structural-tag based tool calling: `off`, + /// Server-side floor for structural-tag based tool calling: `auto`, /// `function`, or `parameter`. #[arg(long, default_value_t)] tool_strict_level: ToolStrictLevel, @@ -248,8 +248,8 @@ pub struct SharedRuntimeArgs { #[serde(default = "default_py_bootstrap_parser_selection")] pub reasoning_parser: ParserSelection, /// Server-side floor for structural-tag based tool calling, applied on - /// top of the per-tool `strict` field: `off` constrains a - /// `tool_choice = "auto"` request only when a tool sets `strict: true`, + /// top of the per-tool `strict` field: `auto` follows the request's + /// tool choice and per-tool strictness, /// `function` constrains the tool-call envelope for every request with /// tools, `parameter` additionally pins argument schemas. #[arg(long, default_value_t)] diff --git a/rust/src/cmd/src/cli/tests.rs b/rust/src/cmd/src/cli/tests.rs index 1cd8940314db..f7f6d0a65815 100644 --- a/rust/src/cmd/src/cli/tests.rs +++ b/rust/src/cmd/src/cli/tests.rs @@ -184,7 +184,7 @@ fn serve_args_forward_python_flags_with_separator() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, - tool_strict_level: Off, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1024,7 +1024,7 @@ fn frontend_args_accept_json() { engine_ready_timeout_secs: 600, tool_call_parser: None, reasoning_parser: None, - tool_strict_level: Off, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1695,7 +1695,7 @@ fn serve_args_accept_handshake_aliases() { engine_ready_timeout_secs: 600, tool_call_parser: Auto, reasoning_parser: Auto, - tool_strict_level: Off, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, max_logprobs: None, @@ -1853,7 +1853,7 @@ fn serve_frontend_config_uses_dp_address_as_advertised_host() { }, tool_call_parser: Auto, reasoning_parser: Auto, - tool_strict_level: Off, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, @@ -1946,7 +1946,7 @@ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() { }, tool_call_parser: Auto, reasoning_parser: Auto, - tool_strict_level: Off, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, @@ -2060,7 +2060,7 @@ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present }, tool_call_parser: None, reasoning_parser: None, - tool_strict_level: Off, + tool_strict_level: Auto, renderer: Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/server/examples/external_engine_openai_qwen.rs b/rust/src/server/examples/external_engine_openai_qwen.rs index 5177658cefa6..3de227babdbe 100644 --- a/rust/src/server/examples/external_engine_openai_qwen.rs +++ b/rust/src/server/examples/external_engine_openai_qwen.rs @@ -71,7 +71,7 @@ async fn main() -> Result<()> { }, tool_call_parser: ParserSelection::Auto, reasoning_parser: ParserSelection::Auto, - tool_strict_level: ToolStrictLevel::Off, + tool_strict_level: ToolStrictLevel::Auto, renderer: RendererSelection::Auto, language_model_only: false, chat_template: None, diff --git a/rust/src/server/src/render.rs b/rust/src/server/src/render.rs index 343e98152cb7..50485e26a408 100644 --- a/rust/src/server/src/render.rs +++ b/rust/src/server/src/render.rs @@ -157,7 +157,7 @@ mod tests { shutdown.cancel(); let error = serve_render( RenderConfig { - tool_strict_level: ToolStrictLevel::Off, + tool_strict_level: ToolStrictLevel::Auto, model: "test-model".to_string(), revision: None, hf_overrides: Default::default(), diff --git a/tests/tool_parsers/test_structural_tag_registry.py b/tests/tool_parsers/test_structural_tag_registry.py index 2a18e2be10f1..c216b398cc5b 100644 --- a/tests/tool_parsers/test_structural_tag_registry.py +++ b/tests/tool_parsers/test_structural_tag_registry.py @@ -879,7 +879,7 @@ def _dumped(tag: StructuralTag) -> str: return json.dumps(tag.model_dump(), ensure_ascii=False) -def test_tool_strict_level_off_is_the_default( +def test_tool_strict_level_auto_is_the_default( sample_tools: list[ChatCompletionToolsParam], ): """Auto + no strict tool gets no tag unless the operator raises the floor.""" @@ -1069,6 +1069,9 @@ def test_tool_strict_level_parameter_overrides_client_strict_false(): def test_tool_strict_level_from_name(): + assert ToolStrictLevel.from_name("AUTO") is ToolStrictLevel.AUTO assert ToolStrictLevel.from_name("Parameter") is ToolStrictLevel.PARAMETER - with pytest.raises(ValueError, match="expected one of off, function, parameter"): + with pytest.raises(ValueError, match="expected one of auto, function, parameter"): ToolStrictLevel.from_name("strict") + with pytest.raises(ValueError, match="expected one of auto, function, parameter"): + ToolStrictLevel.from_name("off") diff --git a/vllm/entrypoints/anthropic/serving.py b/vllm/entrypoints/anthropic/serving.py index d774f1d91e9f..6cc1271add1c 100644 --- a/vllm/entrypoints/anthropic/serving.py +++ b/vllm/entrypoints/anthropic/serving.py @@ -112,7 +112,7 @@ def __init__( reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, - tool_strict_level: str = "off", + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, diff --git a/vllm/entrypoints/cohere/serving.py b/vllm/entrypoints/cohere/serving.py index 8a80141e788b..63d00f0c576a 100644 --- a/vllm/entrypoints/cohere/serving.py +++ b/vllm/entrypoints/cohere/serving.py @@ -249,7 +249,7 @@ def __init__( reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, - tool_strict_level: str = "off", + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, default_chat_template_kwargs: dict[str, Any] | None = None, diff --git a/vllm/entrypoints/launchers/cli_args.py b/vllm/entrypoints/launchers/cli_args.py index 80b545301996..488a7f2c43c8 100644 --- a/vllm/entrypoints/launchers/cli_args.py +++ b/vllm/entrypoints/launchers/cli_args.py @@ -117,10 +117,10 @@ class BaseFrontendArgs: """Special the tool parser plugin write to parse the model-generated tool into OpenAI API format, the name register in this plugin can be used in `--tool-call-parser`.""" - tool_strict_level: ToolStrictLevelName = "off" + tool_strict_level: ToolStrictLevelName = "auto" """Server-side floor for structural-tag based tool calling, applied on top - of the per-tool `strict` field. `off` constrains a `tool_choice="auto"` - request only when a tool sets `strict: true`; `function` constrains the + of the per-tool `strict` field. `auto` follows the request's tool choice + and per-tool strictness; `function` constrains the tool-call envelope (markup and function name) for every request with tools; `parameter` additionally pins argument schemas, as if every tool were `strict: true`.""" diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index ac82ae130d3c..3e2d8f7369ac 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -132,7 +132,7 @@ def __init__( enable_auto_tools: bool = False, exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, - tool_strict_level: str = "off", + tool_strict_level: str = "auto", enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, enable_log_outputs: bool = False, diff --git a/vllm/entrypoints/openai/responses/serving.py b/vllm/entrypoints/openai/responses/serving.py index 9dead9a4f59b..8108c5f77537 100644 --- a/vllm/entrypoints/openai/responses/serving.py +++ b/vllm/entrypoints/openai/responses/serving.py @@ -109,7 +109,7 @@ def __init__( reasoning_parser: str = "", enable_auto_tools: bool = False, tool_parser: str | None = None, - tool_strict_level: str = "off", + tool_strict_level: str = "auto", tool_server: ToolServer | None = None, enable_prompt_tokens_details: bool = False, enable_force_include_usage: bool = False, diff --git a/vllm/parser/abstract_parser.py b/vllm/parser/abstract_parser.py index 16ba6c8427c7..0e84e710ed79 100644 --- a/vllm/parser/abstract_parser.py +++ b/vllm/parser/abstract_parser.py @@ -97,7 +97,7 @@ class Parser: reasoning_parser_cls: type[ReasoningParser] | None = None tool_parser_cls: type[ToolParser] | None = None # Server-side floor for tool-call structural tags (--tool-strict-level). - tool_strict_level: ToolStrictLevel = ToolStrictLevel.OFF + tool_strict_level: ToolStrictLevel = ToolStrictLevel.AUTO def __init__( self, diff --git a/vllm/parser/parser_manager.py b/vllm/parser/parser_manager.py index e8aed9fea736..41b27967a6cb 100644 --- a/vllm/parser/parser_manager.py +++ b/vllm/parser/parser_manager.py @@ -78,7 +78,7 @@ def get_parser( enable_auto_tools: bool = False, model_name: str | None = None, is_harmony: bool = False, - tool_strict_level: str = "off", + tool_strict_level: str = "auto", ) -> type[Parser] | None: """Get a Parser that handles both reasoning and tool parsing. diff --git a/vllm/renderers/online_derenderer.py b/vllm/renderers/online_derenderer.py index b611fad9b627..05f2710afb95 100644 --- a/vllm/renderers/online_derenderer.py +++ b/vllm/renderers/online_derenderer.py @@ -54,7 +54,7 @@ def __init__( exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, reasoning_parser: str | None = None, - tool_strict_level: str = "off", + tool_strict_level: str = "auto", default_chat_template_kwargs: dict[str, Any] | None = None, log_error_stack: bool = False, ) -> None: diff --git a/vllm/renderers/online_renderer.py b/vllm/renderers/online_renderer.py index db4de6799b86..f4d21dedad06 100644 --- a/vllm/renderers/online_renderer.py +++ b/vllm/renderers/online_renderer.py @@ -127,7 +127,7 @@ def __init__( exclude_tools_when_tool_choice_none: bool = False, tool_parser: str | None = None, reasoning_parser: str | None = None, - tool_strict_level: str = "off", + tool_strict_level: str = "auto", default_chat_template_kwargs: dict[str, Any] | None = None, log_error_stack: bool = False, ) -> None: diff --git a/vllm/tool_parsers/abstract_tool_parser.py b/vllm/tool_parsers/abstract_tool_parser.py index 5e597d1195bb..953782daa96f 100644 --- a/vllm/tool_parsers/abstract_tool_parser.py +++ b/vllm/tool_parsers/abstract_tool_parser.py @@ -169,7 +169,7 @@ def get_structural_tag( request: ChatCompletionRequest | ResponsesRequest, *, reasoning: bool = False, - strict_level: ToolStrictLevel = ToolStrictLevel.OFF, + strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, ): if self.structural_tag_model is None: return None diff --git a/vllm/tool_parsers/hy_v4_tool_parser.py b/vllm/tool_parsers/hy_v4_tool_parser.py index f73ade9e27ae..b9c857dbd9e6 100644 --- a/vllm/tool_parsers/hy_v4_tool_parser.py +++ b/vllm/tool_parsers/hy_v4_tool_parser.py @@ -995,7 +995,7 @@ def get_structural_tag( request: ChatCompletionRequest | ResponsesRequest, *, reasoning: bool = False, - strict_level: ToolStrictLevel = ToolStrictLevel.OFF, + strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, ) -> StructuralTag | None: """Build a structural tag matching HYV4's tool tokens. diff --git a/vllm/tool_parsers/structural_tag_registry.py b/vllm/tool_parsers/structural_tag_registry.py index 6f07f250a5ab..5d65b186fe30 100644 --- a/vllm/tool_parsers/structural_tag_registry.py +++ b/vllm/tool_parsers/structural_tag_registry.py @@ -136,7 +136,7 @@ def _resolve_tool_strictness( """ if ( tool_choice == "auto" - and strict_level == ToolStrictLevel.OFF + and strict_level == ToolStrictLevel.AUTO and not _any_tool_strict(tools) ): return None @@ -154,7 +154,7 @@ def get_model_structural_tag( tool_choice: ToolChoice, reasoning: bool, token_suffix: str = "", - strict_level: ToolStrictLevel = ToolStrictLevel.OFF, + strict_level: ToolStrictLevel = ToolStrictLevel.AUTO, ) -> StructuralTag | None: """Build a structural tag with xgrammar's builtin model templates.""" if not tools or tool_choice == "none": diff --git a/vllm/tool_parsers/tool_strict_level.py b/vllm/tool_parsers/tool_strict_level.py index 1485ba5c712f..03a7d8b170a1 100644 --- a/vllm/tool_parsers/tool_strict_level.py +++ b/vllm/tool_parsers/tool_strict_level.py @@ -4,20 +4,20 @@ import enum from typing import Literal -ToolStrictLevelName = Literal["off", "function", "parameter"] +ToolStrictLevelName = Literal["auto", "function", "parameter"] class ToolStrictLevel(enum.IntEnum): """Server-side floor for tool-call structural tags (``--tool-strict-level``). - OFF: constrain a ``tool_choice="auto"`` request only when a tool sets - ``strict: true``. + AUTO: derive call constraints from ``tool_choice`` and argument-schema + enforcement from each tool's explicit ``strict`` field. FUNCTION: constrain the tool-call envelope for every request with tools. PARAMETER: additionally pin argument schemas, as if every tool were ``strict: true``. """ - OFF = 0 + AUTO = 0 FUNCTION = 1 PARAMETER = 2