diff --git a/Cargo.lock b/Cargo.lock index 5415fe434db2..12ec6433df40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2875,9 +2875,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.6" +version = "9.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" +checksum = "a751a31502748c656f69bd1fab01f1d844b6bb629b69e9fbed0b0da9af916b7a" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 6aa0355874f6..da1fc1ab0927 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,7 @@ dynamo-memory = { path = "lib/memory", version = "1.6.0" } dynamo-mocker = { path = "lib/mocker", version = "1.6.0" } dynamo-kv-router = { path = "lib/kv-router", version = "1.6.0" } dynamo-protocols = { version = "=6.1.0" } -dynamo-parsers = { version = "9.2.6" } +dynamo-parsers = { version = "=9.2.7" } # Published on crates.io. To see all available versions: # web: https://crates.io/crates/dynamo-parsers-v2/versions diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/components/src/dynamo/common/configuration/groups/runtime_args.py index 55911146bafe..30970ef35986 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/components/src/dynamo/common/configuration/groups/runtime_args.py @@ -40,8 +40,8 @@ class DynamoRuntimeConfig(ConfigBase): dyn_reasoning_parser: Optional[str] = None dyn_default_thinking_mode: Optional[str] = None exclude_tools_when_tool_choice_none: bool = True - dyn_enable_structural_tag: bool = False - dyn_structural_tag_scope: str = "auto" + dyn_enable_structural_tag: bool = True + dyn_structural_tag_scope: str = "always" dyn_structural_tag_schema: str = "auto" custom_jinja_template: Optional[str] = None endpoint_types: str @@ -293,16 +293,18 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: g, flag_name="--dyn-enable-structural-tag", env_var="DYN_ENABLE_STRUCTURAL_TAG", - default=False, - help="Enable structural tag guided decoding for tool calls. " - "Named Kimi K3 tool_choice requests always activate their required " - "XTML structural tag even when this flag is off.", + default=True, + help="Enable structural tag guided decoding for tool calls when the configured " + "parser and backend support it. Disabling this flag suppresses optional " + "guidance; Rust retains native tags for Kimi K2 required/named and " + "Kimi K3 named tool choices. Python vLLM and SGLang respect the opt-out. " + "Unsupported schema constructs can cause backend grammar compilation errors.", ) add_argument( g, flag_name="--dyn-structural-tag-scope", env_var="DYN_STRUCTURAL_TAG_SCOPE", - default="auto", + default="always", choices=["auto", "always"], help="Controls when structural tags are activated. " "'auto': for required/named tool_choice, and if any tool has strict=true " @@ -317,9 +319,10 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: default="auto", choices=["auto", "strict"], help="Controls parameter schema strictness inside structural tags. " - "'auto': real schema unless the tool explicitly sets strict=false; " - "strict=false tools are syntactically constrained but schema-unconstrained. " - "'strict': real parameter schema for all tools.", + "'auto': declared parameter schema for tools with strict omitted or true; " + "tools with strict=false keep the native tool envelope constrained but use " + "schema-unconstrained arguments. " + "'strict': declared parameter schema for all tools, overriding strict=false.", ) add_argument( g, diff --git a/components/src/dynamo/common/tests/configuration/test_runtime_args.py b/components/src/dynamo/common/tests/configuration/test_runtime_args.py index 5ee112ab9d54..d1a011815c35 100644 --- a/components/src/dynamo/common/tests/configuration/test_runtime_args.py +++ b/components/src/dynamo/common/tests/configuration/test_runtime_args.py @@ -31,6 +31,35 @@ def _parse_runtime_args(argv: list[str]) -> tuple[DynamoRuntimeConfig, str]: return config, parser.format_help() +def test_structural_tags_default_on_for_supported_parsers(monkeypatch): + monkeypatch.delenv("DYN_ENABLE_STRUCTURAL_TAG", raising=False) + monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCOPE", raising=False) + monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCHEMA", raising=False) + + config, _ = _parse_runtime_args([]) + + assert config.dyn_enable_structural_tag is True + assert config.dyn_structural_tag_scope == "always" + assert config.dyn_structural_tag_schema == "auto" + + +def test_structural_tag_schema_help_describes_omitted_strict(monkeypatch): + monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCHEMA", raising=False) + + _, help_text = _parse_runtime_args([]) + + assert "strict omitted or true" in help_text + assert "overriding strict=false" in help_text + + +def test_structural_tag_global_opt_out(monkeypatch): + monkeypatch.setenv("DYN_ENABLE_STRUCTURAL_TAG", "false") + + config, _ = _parse_runtime_args([]) + + assert config.dyn_enable_structural_tag is False + + def test_fpm_trace_defaults_disabled(monkeypatch): monkeypatch.delenv("DYN_FPM_TRACE", raising=False) diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index f7d4010e47a4..83803e97eaa3 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -32,9 +32,22 @@ from dynamo.frontend.vllm_protocol import DeltaFunctionCall, DeltaMessage, DeltaToolCall from dynamo.llm.exceptions import InvalidArgument +from .structural_tag_policy import ( + ToolChoiceKind, + effective_tool_strict, + should_attempt_structural_tag, +) from .thinking import apply_default_thinking_mode_to_template_kwargs from .utils import legacy_guided_decoding +get_model_structural_tag: Any +try: + from vllm.tool_parsers import structural_tag_registry +except ImportError: # Older supported vLLM releases do not expose this registry. + get_model_structural_tag = None +else: + get_model_structural_tag = structural_tag_registry.get_model_structural_tag + if TYPE_CHECKING: from vllm.config import ModelConfig @@ -204,21 +217,32 @@ def _should_build_tool_call_guidance( # enforced guarantee. sglang_prepost.py has the same gap. if tool_choice == "none": return False - if _is_forced_tool_choice(tool_choice): - return True - if structural_tag_mode != "on": - return False - if tool_choice != "auto": - return False - if structural_tag_scope == "always": - return True # An explicit single-call request is enough to attempt structural-tag # guidance. Forced-choice JSON guidance also bounds its array schema below. explicit_single_call = ( "parallel_tool_calls" in request.model_fields_set and request.parallel_tool_calls is False ) - return explicit_single_call or any(_tool_is_strict(tool) for tool in request.tools) + tool_choice_kind: ToolChoiceKind = ( + "required" + if tool_choice == "required" + else "named" + if _is_named_tool_choice(tool_choice) + else "auto" + if tool_choice == "auto" + else "other" + ) + attempt_structural_tag = should_attempt_structural_tag( + mode=structural_tag_mode, + scope=structural_tag_scope, + tool_choice_kind=tool_choice_kind, + has_tools=True, + any_explicit_strict=any(_tool_is_strict(tool) for tool in request.tools), + parallel_tool_calls_explicitly_false=explicit_single_call, + ) + # Forced choices retain the generic JSON fallback when structural tags are + # disabled or unavailable. + return attempt_structural_tag or _is_forced_tool_choice(tool_choice) def _request_for_vllm_structural_tag( @@ -226,13 +250,14 @@ def _request_for_vllm_structural_tag( *, structural_tag_schema: str, ) -> ChatCompletionRequest: - strict_schema = structural_tag_schema == "strict" tools = [ tool.model_copy( update={ "function": tool.function.model_copy( update={ - "strict": True if strict_schema else tool.function.strict, + "strict": effective_tool_strict( + tool.function.strict, structural_tag_schema + ), } ) } @@ -264,7 +289,39 @@ def build_tool_call_guided_decoding( request, structural_tag_schema=structural_tag_schema, ) - structural_tag = tool_parser.get_structural_tag(request_for_tag) + try: + structural_tag_model = getattr(tool_parser, "structural_tag_model", None) + structural_tag_kwargs = {} + if structural_tag_model == "hy_v4" and get_model_structural_tag is not None: + # HYV4's parser detects checkpoint-specific tokens at construction. + # Keep its suffix while Dynamo owns activation of the registry path. + structural_tag_kwargs["token_suffix"] = getattr( + tool_parser, "_extractor" + ).token_suffix + structural_tag = ( + get_model_structural_tag( + model=structural_tag_model, + tools=request_for_tag.tools, + tool_choice=request_for_tag.tool_choice, + reasoning=False, + **structural_tag_kwargs, + ) + if structural_tag_model is not None + and get_model_structural_tag is not None + else tool_parser.get_structural_tag(request_for_tag) + ) + except ( + AttributeError, + KeyError, + NotImplementedError, + RuntimeError, + TypeError, + ValueError, + ): + # Parser implementations are third-party capability providers. A + # failed structural-tag build must preserve the pre-existing JSON + # or unconstrained compatibility path for the request. + structural_tag = None if structural_tag is not None: tag_value = ( structural_tag.model_dump() diff --git a/components/src/dynamo/frontend/sglang_prepost.py b/components/src/dynamo/frontend/sglang_prepost.py index b34e8d6f609a..2c295da24704 100644 --- a/components/src/dynamo/frontend/sglang_prepost.py +++ b/components/src/dynamo/frontend/sglang_prepost.py @@ -33,6 +33,11 @@ from dynamo.common.utils.guided_json import admits_only_empty_object from dynamo.llm.exceptions import InvalidArgument +from .structural_tag_policy import ( + ToolChoiceKind, + effective_tool_strict, + should_attempt_structural_tag, +) from .thinking import apply_default_thinking_mode_to_template_kwargs from .utils import PreprocessError, legacy_guided_decoding, random_call_id @@ -204,13 +209,24 @@ def _client_wants_separate_reasoning(request: dict[str, Any]) -> bool: return bool(value) -def convert_tools(tools: list[dict[str, Any]] | None) -> list[SglangTool] | None: - """Convert OpenAI tool dicts to SGLang Tool objects.""" +def convert_tools( + tools: list[dict[str, Any]] | None, + *, + structural_tag_schema: str | None = None, +) -> list[SglangTool] | None: + """Convert OpenAI tool dicts to SGLang Tool objects. + + ``structural_tag_schema`` is set only for the guidance copy. Prompt and + parser copies retain SGLang's existing omitted-strict representation. + """ if not tools: return None sglang_tools = [] for tool in tools: func = tool.get("function", {}) + strict = func.get("strict", False) + if structural_tag_schema is not None: + strict = effective_tool_strict(func.get("strict"), structural_tag_schema) sglang_tools.append( SglangTool( type=tool.get("type", "function"), @@ -218,7 +234,7 @@ def convert_tools(tools: list[dict[str, Any]] | None) -> list[SglangTool] | None name=func.get("name", ""), description=func.get("description"), parameters=func.get("parameters"), - strict=func.get("strict", False), + strict=strict, ), ) ) @@ -285,6 +301,7 @@ def create_parsers( reasoning_parser_name: str | None, sglang_tools: list[SglangTool] | None = None, force_reasoning: bool = False, + guided_decoding: dict[str, Any] | None = None, tokenizer: Any | None = None, ) -> tuple[ToolCallParserType | None, ReasoningParser | None]: """Create tool call and reasoning parsers for a request. @@ -295,10 +312,9 @@ def create_parsers( If ``sglang_tools`` is provided, reuses them; otherwise converts from the request's ``tools`` field. - For ``tool_choice="required"`` or a named function, uses - :class:`JsonArrayParser` (matching native SGLang) since guided decoding - constrains the output to a JSON array. Otherwise uses the model-specific - :class:`FunctionCallParser`. + Required and named choices use the model-specific :class:`FunctionCallParser` + when the effective guidance is a structural tag. Their JSON fallback keeps + :class:`JsonArrayParser`. Automatic choices use :class:`FunctionCallParser`. """ if sglang_tools is None: sglang_tools = convert_tools(request.get("tools")) @@ -306,7 +322,9 @@ def create_parsers( tool_call_parser: ToolCallParserType | None = None if sglang_tools and tool_choice != "none": - if tool_choice == "required" or _is_named_tool_choice(tool_choice): + if (tool_choice == "required" or _is_named_tool_choice(tool_choice)) and not ( + guided_decoding is not None and "structural_tag" in guided_decoding + ): tool_call_parser = JsonArrayParser() elif tool_call_parser_name: tool_call_parser_name = _normalize_sglang_parser_name(tool_call_parser_name) @@ -587,6 +605,29 @@ def _call_with_optional_parallel_tool_calls( return func(*args) +def _call_structure_constraint( + func: Any, + *args: Any, + parallel_tool_calls: Any, + thinking_mode: bool, +) -> Any: + """Call SGLang parser APIs across supported signature versions.""" + kwargs: dict[str, Any] = {} + if _callable_accepts_kwarg(func, "parallel_tool_calls"): + kwargs["parallel_tool_calls"] = parallel_tool_calls + if _callable_accepts_kwarg(func, "thinking_mode"): + kwargs["thinking_mode"] = thinking_mode + return func(*args, **kwargs) + + +def _enforce_function_strict_level(parser: FunctionCallParser) -> None: + """Keep SGLang's native tool envelope active for policy-selected requests.""" + current_level = getattr(parser, "tool_strict_level", None) + function_level = getattr(type(current_level), "FUNCTION", None) + if function_level is not None and current_level < function_level: + parser.tool_strict_level = function_level + + def _parser_tokenizer_kwargs(parser: Any, tokenizer: Any) -> dict[str, Any]: if tokenizer is not None and _callable_accepts_kwarg(parser, "tokenizer"): return {"tokenizer": tokenizer} @@ -599,6 +640,9 @@ def build_tool_call_guided_decoding( tool_call_parser_name: str | None, tokenizer: Any | None = None, sglang_tools: list[SglangTool] | None, + structural_tag_mode: str = "off", + structural_tag_scope: str = "auto", + structural_tag_schema: str = "auto", ) -> dict[str, Any] | None: """Build native-SGLang-like tool call constraints for guided decoding.""" if not sglang_tools: @@ -615,7 +659,88 @@ def build_tool_call_guided_decoding( parallel_tool_calls = True constraint: Any = None - if tool_choice == "required" or _is_named_tool_choice(tool_choice): + is_named_choice = _is_named_tool_choice(tool_choice) + is_forced_choice = tool_choice == "required" or is_named_choice + sglang_tool_choice: Any = tool_choice + if is_named_choice: + sglang_tool_choice = SglangToolChoice( + type="function", + function=SglangToolChoiceFuncName( + name=tool_choice["function"]["name"], + ), + ) + tool_choice_kind: ToolChoiceKind = ( + "required" + if tool_choice == "required" + else "named" + if is_named_choice + else "auto" + if tool_choice == "auto" + else "other" + ) + raw_tools = request.get("tools") or [] + attempt_structural_tag = should_attempt_structural_tag( + mode=structural_tag_mode, + scope=structural_tag_scope, + tool_choice_kind=tool_choice_kind, + has_tools=bool(raw_tools), + any_explicit_strict=any( + tool.get("function", {}).get("strict") is True for tool in raw_tools + ), + parallel_tool_calls_explicitly_false=( + "parallel_tool_calls" in request and parallel_tool_calls is False + ), + ) + + if attempt_structural_tag and tool_call_parser_name: + guidance_tools = convert_tools( + raw_tools, + structural_tag_schema=structural_tag_schema, + ) + if is_named_choice: + # SGLang's legacy tag builder ignores the name in tool_choice. + guidance_tools = [ + tool + for tool in guidance_tools or [] + if tool.function.name == sglang_tool_choice.function.name + ] + tool_call_parser_name = _normalize_sglang_parser_name(tool_call_parser_name) + try: + parser = FunctionCallParser( + tools=guidance_tools, + tool_call_parser=tool_call_parser_name, + **_parser_tokenizer_kwargs(FunctionCallParser, tokenizer), + ) + # SGLang's AUTO level returns no structural tag when every tool is + # strict:false. Dynamo's policy still requires the function envelope; + # per-tool strict flags continue to control argument enforcement. + _enforce_function_strict_level(parser) + constraint = _call_structure_constraint( + parser.get_structure_constraint, + sglang_tool_choice, + parallel_tool_calls=parallel_tool_calls, + # The backend's reasoning gate consumes the reasoning prefix; + # this grammar starts with the tool payload after that boundary. + thinking_mode=False, + ) + except ( + AttributeError, + KeyError, + NotImplementedError, + RuntimeError, + TypeError, + ValueError, + ): + # This is a third-party capability probe. Unsupported parser/schema + # combinations retain the existing forced-choice JSON fallback or, + # for auto, unconstrained generation. + constraint = None + + guided_decoding = _serialize_tool_constraint(constraint) + if guided_decoding is not None: + return guided_decoding + + if is_forced_choice: if named_closed_zero_arg_tool(request) is not None: return {"regex": r"\{\}"} @@ -623,14 +748,6 @@ def build_tool_call_guided_decoding( # ToolChoice) for the named-function case — passing our raw dict # would silently fall through and return None, disabling guided # decoding and letting the model omit required fields. - sglang_tool_choice: Any = tool_choice - if _is_named_tool_choice(tool_choice): - sglang_tool_choice = SglangToolChoice( - type="function", - function=SglangToolChoiceFuncName( - name=tool_choice["function"]["name"], - ), - ) constraint = ( "json_schema", _call_with_optional_parallel_tool_calls( @@ -640,43 +757,24 @@ def build_tool_call_guided_decoding( parallel_tool_calls=parallel_tool_calls, ), ) - # TODO: this applies a structural-tag constraint for tool_choice="auto" - # whenever a tool-call parser is configured, and reads NONE of - # structural_tag_mode / structural_tag_scope / structural_tag_schema. Those - # knobs are accepted on the CLI and published into the model card by - # sglang/register.py, so an operator setting the mode to "off" still gets a - # constraint on this path. prepost.py requires structural_tag_mode == "on" - # (_should_build_tool_call_guidance) and preprocessor/structural_tag.rs - # requires StructuralTagMode != Off, so at the default mode ("off") the same - # request is unconstrained on both of those paths and constrained here. - # Also unlike them, "auto" is never gated on scope/strict, so this behaves as - # scope="always" with no way to narrow it. - elif tool_call_parser_name: - tool_call_parser_name = _normalize_sglang_parser_name(tool_call_parser_name) - parser = FunctionCallParser( - tools=sglang_tools, - tool_call_parser=tool_call_parser_name, - **_parser_tokenizer_kwargs(FunctionCallParser, tokenizer), - ) - constraint = _call_with_optional_parallel_tool_calls( - parser.get_structure_constraint, - tool_choice, - parallel_tool_calls=parallel_tool_calls, - ) + return _serialize_tool_constraint(constraint) - if isinstance(constraint, tuple) and len(constraint) == 2: - if constraint[0] == "json_schema": - return {"json": constraint[1]} - if constraint[0] == "structural_tag": - tag_value = constraint[1] - # SGLang returns a Pydantic model (LegacyStructuralTagResponseFormat) - # here. Convert to a plain dict before it hits the RPC layer — - # msgpack/serde_json cannot serialize BaseModel instances. - if hasattr(tag_value, "model_dump"): - tag_value = tag_value.model_dump() - return {"structural_tag": tag_value} - return None +def _serialize_tool_constraint(constraint: Any) -> dict[str, Any] | None: + if not (isinstance(constraint, tuple) and len(constraint) == 2): + return None + if constraint[0] == "json_schema": + return {"json": constraint[1]} + if constraint[0] != "structural_tag": + return None + + tag_value = constraint[1] + # SGLang returns a Pydantic model (LegacyStructuralTagResponseFormat) + # here. Convert to a plain dict before it hits the RPC layer because + # msgpack/serde_json cannot serialize BaseModel instances. + if hasattr(tag_value, "model_dump"): + tag_value = tag_value.model_dump() + return {"structural_tag": tag_value} def build_response_format_guided_decoding( @@ -773,6 +871,9 @@ def preprocess_chat_request( exclude_tools_when_tool_choice_none: bool = True, template_force_reasoning: bool = False, default_thinking_mode: str | None = None, + structural_tag_mode: str = "off", + structural_tag_scope: str = "auto", + structural_tag_schema: str = "auto", ) -> SglangPreprocessResult: """Preprocess a chat request using SGLang tokenizer and parser APIs. @@ -876,21 +977,14 @@ def preprocess_chat_request( raise PreprocessError(str(exc)) from exc prompt_token_ids = _normalize_prompt_token_ids(rendered) - # Build parsers after rendering, so DeepSeek-V4 can use its custom encoder - # while still sharing the existing Dynamo parser/guided-decoding behavior. - tool_call_parser, reasoning_parser = create_parsers( - request, - tool_call_parser_name=tool_call_parser_name, - reasoning_parser_name=effective_reasoning_parser_name, - sglang_tools=sglang_tools, - force_reasoning=force_reasoning, - tokenizer=tokenizer, - ) response_format_guided_decoding = build_response_format_guided_decoding(request) tool_call_guided_decoding = build_tool_call_guided_decoding( request, tool_call_parser_name=tool_call_parser_name, sglang_tools=sglang_tools, + structural_tag_mode=structural_tag_mode, + structural_tag_scope=structural_tag_scope, + structural_tag_schema=structural_tag_schema, tokenizer=tokenizer, ) # This path also never reads the legacy guided_json / guided_regex / @@ -938,6 +1032,17 @@ def preprocess_chat_request( or tool_call_guided_decoding ) + # Match the parser to the installed format, including forced-choice fallback. + tool_call_parser, reasoning_parser = create_parsers( + request, + tool_call_parser_name=tool_call_parser_name, + reasoning_parser_name=effective_reasoning_parser_name, + sglang_tools=sglang_tools, + force_reasoning=force_reasoning, + guided_decoding=guided_decoding, + tokenizer=tokenizer, + ) + return SglangPreprocessResult( prompt_token_ids=prompt_token_ids, tool_call_parser=tool_call_parser, diff --git a/components/src/dynamo/frontend/sglang_processor.py b/components/src/dynamo/frontend/sglang_processor.py index e7177e029f8c..22c0525f4265 100644 --- a/components/src/dynamo/frontend/sglang_processor.py +++ b/components/src/dynamo/frontend/sglang_processor.py @@ -40,6 +40,7 @@ preprocess_chat_request, resolve_skip_special_tokens, ) +from .structural_tag_policy import runtime_structural_tag_options from .thinking import runtime_default_thinking_mode from .utils import ( PreprocessError, @@ -272,6 +273,9 @@ def _map_finish_reason(raw: str | None) -> str | None: _w_exclude_tools_when_tool_choice_none: bool = True _w_template_force_reasoning: bool = False _w_default_thinking_mode: str | None = None +_w_structural_tag_mode: str = "off" +_w_structural_tag_scope: str = "auto" +_w_structural_tag_schema: str = "auto" def _load_chat_template(chat_template: str | None) -> str | None: @@ -323,11 +327,15 @@ def _init_worker( template_force_reasoning: bool = False, chat_template: str | None = None, default_thinking_mode: str | None = None, + structural_tag_mode: str = "off", + structural_tag_scope: str = "auto", + structural_tag_schema: str = "auto", ) -> None: """Initialize a worker process with its own tokenizer.""" global _w_tokenizer, _w_tool_call_parser_name, _w_reasoning_parser_name global _w_exclude_tools_when_tool_choice_none, _w_template_force_reasoning - global _w_default_thinking_mode + global _w_default_thinking_mode, _w_structural_tag_mode + global _w_structural_tag_scope, _w_structural_tag_schema _w_tokenizer = _load_tokenizer(model_path, trust_remote_code) if chat_template is not None: _w_tokenizer.chat_template = chat_template @@ -336,6 +344,9 @@ def _init_worker( _w_exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none _w_template_force_reasoning = template_force_reasoning _w_default_thinking_mode = default_thinking_mode + _w_structural_tag_mode = structural_tag_mode + _w_structural_tag_scope = structural_tag_scope + _w_structural_tag_schema = structural_tag_schema def _preprocess_worker( @@ -352,6 +363,9 @@ def _preprocess_worker( exclude_tools_when_tool_choice_none=_w_exclude_tools_when_tool_choice_none, template_force_reasoning=_w_template_force_reasoning, default_thinking_mode=_w_default_thinking_mode, + structural_tag_mode=_w_structural_tag_mode, + structural_tag_scope=_w_structural_tag_scope, + structural_tag_schema=_w_structural_tag_schema, ) n = request.get("n", 1) @@ -525,6 +539,9 @@ def __init__( preprocess_workers: int = 0, stream_interval: int = 1, default_thinking_mode: str | None = None, + structural_tag_mode: str = "off", + structural_tag_scope: str = "auto", + structural_tag_schema: str = "auto", ): self.tokenizer = tokenizer # Detect force_reasoning once from the chat template, matching @@ -549,6 +566,9 @@ def __init__( self.debug_perf = debug_perf self.stream_interval = stream_interval self.default_thinking_mode = default_thinking_mode + self.structural_tag_mode = structural_tag_mode + self.structural_tag_scope = structural_tag_scope + self.structural_tag_schema = structural_tag_schema self.preprocess_pool = preprocess_pool if preprocess_pool is not None: self._worker_semaphore: asyncio.Semaphore | None = asyncio.Semaphore( @@ -606,6 +626,9 @@ async def _generator_inner( exclude_tools_when_tool_choice_none=self.exclude_tools_when_tool_choice_none, template_force_reasoning=self.template_force_reasoning, default_thinking_mode=self.default_thinking_mode, + structural_tag_mode=self.structural_tag_mode, + structural_tag_scope=self.structural_tag_scope, + structural_tag_schema=self.structural_tag_schema, ) if self.debug_perf: @@ -710,6 +733,9 @@ async def _generator_inner_pool( tool_call_parser_name=self.tool_call_parser_name, reasoning_parser_name=preproc_result.effective_reasoning_parser_name, force_reasoning=preproc_result.force_reasoning, + guided_decoding=preproc_result.dynamo_preproc["sampling_options"][ + "guided_decoding" + ], tokenizer=self.tokenizer, ) @@ -1067,6 +1093,11 @@ async def chat_engine_factory( or _runtime_config_parser_name(mdc, "reasoning_parser") ) default_thinking_mode = runtime_default_thinking_mode(mdc.runtime_config()) + ( + structural_tag_mode, + structural_tag_scope, + structural_tag_schema, + ) = runtime_structural_tag_options(mdc.runtime_config()) if tool_call_parser_name: logger.info("SGLang tool call parser: %s", tool_call_parser_name) @@ -1095,6 +1126,9 @@ async def chat_engine_factory( template_force_reasoning, chat_template, default_thinking_mode, + structural_tag_mode, + structural_tag_scope, + structural_tag_schema, ), ) futures = [ @@ -1131,6 +1165,9 @@ async def chat_engine_factory( preprocess_workers=preprocess_workers, stream_interval=self.stream_interval, default_thinking_mode=default_thinking_mode, + structural_tag_mode=structural_tag_mode, + structural_tag_scope=structural_tag_scope, + structural_tag_schema=structural_tag_schema, ) gen.exclude_tools_when_tool_choice_none = ( self.config.exclude_tools_when_tool_choice_none diff --git a/components/src/dynamo/frontend/structural_tag_policy.py b/components/src/dynamo/frontend/structural_tag_policy.py new file mode 100644 index 000000000000..b54b7499eaca --- /dev/null +++ b/components/src/dynamo/frontend/structural_tag_policy.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, Literal + +ToolChoiceKind = Literal["none", "auto", "required", "named", "other"] + + +def runtime_structural_tag_options( + runtime_config: Any, +) -> tuple[str, str, str]: + """Read structural-tag policy from a model-card runtime config.""" + if not isinstance(runtime_config, dict): + return "off", "auto", "auto" + return ( + runtime_config.get("structural_tag_mode", "off"), + runtime_config.get("structural_tag_scope", "auto"), + runtime_config.get("structural_tag_schema", "auto"), + ) + + +def should_attempt_structural_tag( + *, + mode: str, + scope: str, + tool_choice_kind: ToolChoiceKind, + has_tools: bool, + any_explicit_strict: bool, + parallel_tool_calls_explicitly_false: bool, +) -> bool: + """Return whether frontend preprocessing should request a structural tag.""" + if mode != "on" or not has_tools or tool_choice_kind == "none": + return False + if tool_choice_kind in {"required", "named"}: + return True + if tool_choice_kind != "auto": + return False + if scope == "always": + return True + return any_explicit_strict or parallel_tool_calls_explicitly_false + + +def effective_tool_strict(request_strict: bool | None, schema_mode: str) -> bool: + """Resolve request strictness for grammar construction. + + In ``auto`` schema mode, omitted ``strict`` is schema-enforced and only an + explicit ``strict: false`` opts out. ``strict`` schema mode overrides that + request-level opt-out. + """ + return schema_mode == "strict" or request_strict is not False diff --git a/components/src/dynamo/frontend/tests/test_pinned_vllm_structural_tag_registry.py b/components/src/dynamo/frontend/tests/test_pinned_vllm_structural_tag_registry.py new file mode 100644 index 000000000000..86897daddd2d --- /dev/null +++ b/components/src/dynamo/frontend/tests/test_pinned_vllm_structural_tag_registry.py @@ -0,0 +1,58 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pin the real vLLM structural-tag behavior used by Dynamo's auto tool path.""" + +import pytest + +try: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag +except ImportError: + pytest.skip("requires the vLLM structural-tag registry", allow_module_level=True) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.vllm, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +def test_pinned_vllm_registry_auto_floor(): + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + "strict": False, + }, + } + non_strict = ChatCompletionToolsParam.model_validate(tool) + assert ( + get_model_structural_tag( + model="kimi_k3", + tools=[non_strict], + tool_choice="auto", + reasoning=False, + ) + is None + ) + + tool["function"]["strict"] = True + strict = ChatCompletionToolsParam.model_validate(tool) + assert ( + get_model_structural_tag( + model="kimi_k3", + tools=[strict], + tool_choice="auto", + reasoning=False, + ) + is not None + ) diff --git a/components/src/dynamo/frontend/tests/test_sglang_checkpoint_templates.py b/components/src/dynamo/frontend/tests/test_sglang_checkpoint_templates.py index 6f930eb58ea9..8c26eae7ac21 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_checkpoint_templates.py +++ b/components/src/dynamo/frontend/tests/test_sglang_checkpoint_templates.py @@ -143,5 +143,7 @@ def constraint(parser, tool_choice): tokenizer=HunyuanTokenizer(), tool_call_parser_name="hunyuan", reasoning_parser_name="hunyuan", + structural_tag_mode="on", + structural_tag_scope="always", ) assert result.guided_decoding == {"structural_tag": {"type": "object"}} diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index b382f8ea118b..f5e9a84340ba 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -16,8 +16,10 @@ import types from concurrent.futures import Future, ThreadPoolExecutor from contextlib import nullcontext +from enum import IntEnum import pytest +import xgrammar as xgr from _routed_engine_fakes import FakeRoutedEngine, FakeRoutedItem from _thinking_parity import RESOLVED_DISABLED, RESOLVED_ENABLED, THINKING_PARITY_CASES from _tool_guidance_parity import ( @@ -28,6 +30,11 @@ tool_choice_value, ) from jinja2.exceptions import TemplateError, UndefinedError +from sglang.srt.constrained.reasoner_grammar_backend import ReasonerGrammarObject +from sglang.srt.constrained.xgrammar_backend import ( + XGrammarGrammar, + XGrammarGrammarBackend, +) from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.utils.hf_transformers_utils import get_tokenizer @@ -93,6 +100,20 @@ def tokenizer(): return get_tokenizer(MODEL) +@pytest.fixture(scope="module") +def tool_grammar_compiler(tokenizer): + return xgr.GrammarCompiler( + xgr.TokenizerInfo.from_huggingface(tokenizer), max_threads=1 + ) + + +@pytest.fixture(scope="module") +def tool_grammar_backend(tokenizer): + backend = XGrammarGrammarBackend(tokenizer, vocab_size=len(tokenizer)) + yield backend + backend.executor.shutdown(wait=True) + + @pytest.fixture(scope="module") def byte_fallback_tokenizer(): return get_tokenizer(BYTE_FALLBACK_MODEL) @@ -1281,6 +1302,10 @@ def apply_chat_template(self, messages, **kwargs): sglang_processor_module, "_w_exclude_tools_when_tool_choice_none", True ) monkeypatch.setattr(sglang_processor_module, "_w_template_force_reasoning", False) + # Exercise enabled guidance independently of worker initialization or test order. + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_mode", "on") + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_scope", "always") + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_schema", "auto") request = { "model": "moonshotai/Kimi-K3", @@ -1770,6 +1795,239 @@ def test_structural_tag_builds_guided_decoding(self): ) == {"structural_tag": response_format} +@pytest.mark.core +@pytest.mark.timeout(60) +@pytest.mark.parametrize("use_pool", [False, True], ids=["inline", "pool"]) +@pytest.mark.parametrize( + "tool_choice", + [ + pytest.param("required", id="required"), + pytest.param( + {"type": "function", "function": {"name": "get_weather"}}, id="named" + ), + ], +) +def test_forced_structural_tag_returns_tool_call( + tokenizer, monkeypatch, use_pool, tool_choice +): + """Native forced-tool guidance must be decoded into an API tool call.""" + request = { + "model": MODEL, + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "chat_template_kwargs": {"enable_thinking": False}, + "stream": True, + "tool_choice": tool_choice, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ], + } + native_call = ( + "\n\n" + "\nParis\n\n" + "\n" + ) + engine = FakeRoutedEngine( + items=[{"token_ids": tokenizer.encode(native_call), "finish_reason": "stop"}] + ) + + if use_pool: + # Exercise the real worker and parser reconstruction without a subprocess. + monkeypatch.setattr(sglang_processor_module, "_w_tokenizer", tokenizer) + monkeypatch.setattr( + sglang_processor_module, "_w_tool_call_parser_name", "qwen3_coder" + ) + monkeypatch.setattr(sglang_processor_module, "_w_reasoning_parser_name", None) + monkeypatch.setattr( + sglang_processor_module, "_w_exclude_tools_when_tool_choice_none", True + ) + monkeypatch.setattr( + sglang_processor_module, "_w_template_force_reasoning", False + ) + monkeypatch.setattr(sglang_processor_module, "_w_default_thinking_mode", None) + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_mode", "on") + monkeypatch.setattr( + sglang_processor_module, "_w_structural_tag_scope", "always" + ) + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_schema", "auto") + + with ThreadPoolExecutor(max_workers=1) if use_pool else nullcontext() as pool: + processor = SglangProcessor( + tokenizer=tokenizer, + routed_engine=engine, + tool_call_parser_name="qwen3_coder", + reasoning_parser_name=None, + eos_token_ids=None, + preprocess_pool=pool, + preprocess_workers=1 if use_pool else 0, + structural_tag_mode="on", + structural_tag_scope="always", + ) + + async def collect(): + return [item async for item in processor.generator(request)] + + output = asyncio.run(collect()) + + # Check the actual installed guidance, not just the requested policy. + assert "structural_tag" in engine.requests[0]["sampling_options"]["guided_decoding"] + choices = [choice for item in output for choice in item["data"]["choices"]] + calls = [ + call for choice in choices for call in choice["delta"].get("tool_calls", []) + ] + assert calls, output + assert {call["index"] for call in calls} == {0} + assert [ + call["function"]["name"] for call in calls if call["function"].get("name") + ] == ["get_weather"] + assert json.loads( + "".join(call["function"].get("arguments", "") for call in calls) + ) == {"city": "Paris"} + assert not any(choice["delta"].get("content") for choice in choices) + assert choices[-1]["finish_reason"] == "tool_calls" + + +@pytest.mark.core +@pytest.mark.timeout(60) +@pytest.mark.parametrize("tool_choice", ["auto", "required", "named"]) +@pytest.mark.parametrize("thinking", [False, True], ids=["chat", "thinking"]) +@pytest.mark.parametrize( + "separate_reasoning", [False, True], ids=["combined", "separate"] +) +def test_tool_structural_tag_enforced_after_one_reasoning_boundary( + tokenizer, tool_grammar_compiler, tool_choice, thinking, separate_reasoning +): + """The backend consumes reasoning; the inner grammar must enforce tools next.""" + request = { + "model": MODEL, + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "tools": [parity_tool()], + "tool_choice": tool_choice_value(tool_choice), + "chat_template_kwargs": {"enable_thinking": thinking}, + "separate_reasoning": separate_reasoning, + } + pre = preprocess_chat_request( + request, + tokenizer=tokenizer, + tool_call_parser_name="qwen3_coder", + reasoning_parser_name="qwen3", + structural_tag_mode="on", + structural_tag_scope="always", + ) + assert pre.force_reasoning is thinking + require_reasoning = _guided_output_requires_reasoning( + request, pre.force_reasoning, "qwen3", pre.guided_decoding + ) + assert require_reasoning is thinking + assert pre.guided_decoding is not None + compiled = tool_grammar_compiler.compile_structural_tag( + pre.guided_decoding["structural_tag"] + ) + + def grammar_after_reasoning(): + inner = XGrammarGrammar( + matcher=xgr.GrammarMatcher(compiled), + vocab_size=compiled.tokenizer_info.vocab_size, + ctx=compiled, + override_stop_tokens=None, + ) + grammar = ReasonerGrammarObject( + inner, think_end_ids=tokenizer.encode("", add_special_tokens=False) + ) + grammar.maybe_init_reasoning(require_reasoning) + if thinking: + for token in tokenizer.encode( + "I should check the weather.", add_special_tokens=False + ): + grammar.accept_token(token) + # The wrapper consumes the prefix, including the end marker. + assert inner.accepted_tokens == [] + return grammar + + native_call = ( + "\n\n" + "\nParis\n\n" + "\n" + ) + grammar = grammar_after_reasoning() + for token in tokenizer.encode(native_call, add_special_tokens=False): + grammar.accept_token(token) + grammar.accept_token(tokenizer.eos_token_id) + assert grammar.is_terminated() + + # A second reasoning prefix must not leave an undeclared tool unconstrained. + grammar = grammar_after_reasoning() + unknown_call = native_call.replace("get_weather", "undeclared_tool") + with pytest.raises(ValueError, match="Tokens not accepted"): + for token in tokenizer.encode(unknown_call, add_special_tokens=False): + grammar.accept_token(token) + + +@pytest.mark.core +@pytest.mark.timeout(60) +@pytest.mark.filterwarnings( + "ignore:max_rollback_tokens is deprecated.*:DeprecationWarning" +) +@pytest.mark.parametrize("parser_name", ["qwen25", "qwen3_coder"]) +@pytest.mark.parametrize("tool_choice", ["auto", "required", "named"]) +def test_structural_tag_respects_named_tool_choice( + tool_grammar_backend, parser_name, tool_choice +): + """A named choice excludes other declared tools, including legacy tags.""" + raw_tools = [parity_tool(), parity_tool()] + raw_tools[1]["function"]["name"] = "get_forecast" + request = { + "tools": raw_tools, + "tool_choice": ( + {"type": "function", "function": {"name": "get_forecast"}} + if tool_choice == "named" + else tool_choice + ), + } + guided = build_tool_call_guided_decoding( + request, + tool_call_parser_name=parser_name, + sglang_tools=convert_tools(raw_tools), + structural_tag_mode="on", + structural_tag_scope="always", + ) + assert guided is not None + grammar = tool_grammar_backend.dispatch_structural_tag( + json.dumps(guided["structural_tag"]) + ) + assert isinstance(grammar, XGrammarGrammar) + + def accepts_call(name): + if parser_name == "qwen25": + call = ( + '\n{"name":"' + name + '", "arguments":' + '{"city":"Paris"}}\n' + ) + else: + call = ( + f"\n\n" + "\nParis\n\n" + "\n" + ) + return grammar.copy().matcher.accept_string(call) + + assert accepts_call("get_forecast") + assert accepts_call("get_weather") is (tool_choice != "named") + assert [tool["function"]["name"] for tool in raw_tools] == [ + "get_weather", + "get_forecast", + ] + + class TestBuildToolCallGuidedDecoding: # FRONTEND.3 — guided-decoding setup for tool_choice # Keep SGLang's guidance decisions aligned with the shared backend matrix. @pytest.mark.parametrize( @@ -1798,6 +2056,8 @@ def test_shared_tool_guidance_policy(self, tokenizer, case): tokenizer=tokenizer, tool_call_parser_name="kimi_k2", reasoning_parser_name=None, + structural_tag_mode="on", + structural_tag_scope="always", ) assert ( @@ -1843,17 +2103,16 @@ def test_none_when_tool_choice_none(self): def test_auto_tool_guidance_normalizes_minimax_m3_alias( self, monkeypatch, tokenizer ): - tools = convert_tools( - [ - { - "type": "function", - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {}}, - }, - } - ] - ) + raw_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + tools = convert_tools(raw_tools) seen = {} # Test double: capture the parser name used for guided decoding setup. @@ -1872,9 +2131,11 @@ def get_structure_constraint(self, tool_choice, **kwargs): ) guided = build_tool_call_guided_decoding( - {"tool_choice": "auto"}, + {"tool_choice": "auto", "tools": raw_tools}, tool_call_parser_name="minimax_m3_nom", sglang_tools=tools, + structural_tag_mode="on", + structural_tag_scope="always", tokenizer=tokenizer, ) @@ -1992,21 +2253,20 @@ def old_get_json_schema_constraint(sglang_tools, tool_choice): def test_auto_tool_choice_supports_older_structure_constraint_signature( self, monkeypatch ): - tools = convert_tools( - [ - { - "type": "function", - "function": { - "name": "get_weather", - "strict": True, - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - }, + raw_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, }, - } - ] - ) + }, + } + ] + tools = convert_tools(raw_tools) class OldFunctionCallParser: def __init__(self, *, tools, tool_call_parser): @@ -2024,40 +2284,165 @@ def get_structure_constraint(self, tool_choice): ) guided = build_tool_call_guided_decoding( - {"tool_choice": "auto", "parallel_tool_calls": False}, + { + "tool_choice": "auto", + "parallel_tool_calls": False, + "tools": raw_tools, + }, tool_call_parser_name="kimi_k2", sglang_tools=tools, + structural_tag_mode="on", ) assert guided == {"structural_tag": {"type": "object"}} def test_auto_strict_tools_can_build_structural_tag_guidance(self): - tools = convert_tools( - [ - { - "type": "function", - "function": { - "name": "get_weather", - "strict": True, - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, + raw_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], }, - } - ] - ) + }, + } + ] + tools = convert_tools(raw_tools) guided = build_tool_call_guided_decoding( - {"tool_choice": "auto"}, + {"tool_choice": "auto", "tools": raw_tools}, tool_call_parser_name="kimi_k2", sglang_tools=tools, + structural_tag_mode="on", + structural_tag_scope="auto", ) assert isinstance(guided, dict) assert "structural_tag" in guided + @pytest.mark.parametrize( + ("request_strict", "include_strict", "schema_mode", "expected_strict"), + [ + (None, False, "auto", True), + (True, True, "auto", True), + (False, True, "auto", False), + (False, True, "strict", True), + ], + ) + def test_structural_tag_guidance_uses_effective_schema_policy( + self, + monkeypatch, + request_strict, + include_strict, + schema_mode, + expected_strict, + ): + function = { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + } + if include_strict: + function["strict"] = request_strict + raw_tools = [{"type": "function", "function": function}] + seen = {} + + class FakeToolStrictLevel(IntEnum): + AUTO = 0 + FUNCTION = 1 + + class CapturingFunctionCallParser: + def __init__(self, *, tools, tool_call_parser): + del tool_call_parser + seen["strict"] = tools[0].function.strict + self.tool_strict_level = FakeToolStrictLevel.AUTO + + def get_structure_constraint( + self, tool_choice, *, parallel_tool_calls=None, thinking_mode=False + ): + del tool_choice, parallel_tool_calls + seen["thinking_mode"] = thinking_mode + seen["strict_level"] = self.tool_strict_level.name + return "structural_tag", {"type": "object"} + + monkeypatch.setattr( + sglang_prepost_module, + "FunctionCallParser", + CapturingFunctionCallParser, + ) + + guided = build_tool_call_guided_decoding( + {"tool_choice": "auto", "tools": raw_tools}, + tool_call_parser_name="kimi_k2", + sglang_tools=convert_tools(raw_tools), + structural_tag_mode="on", + structural_tag_scope="always", + structural_tag_schema=schema_mode, + ) + + assert guided == {"structural_tag": {"type": "object"}} + assert seen == { + "strict": expected_strict, + "thinking_mode": False, + "strict_level": "FUNCTION", + } + assert function.get("strict") is request_strict + + def test_mode_off_does_not_probe_auto_structural_tag(self, monkeypatch): + raw_tools = [parity_tool()] + + class UnexpectedFunctionCallParser: + def __init__(self, **kwargs): + raise AssertionError(f"unexpected structural-tag probe: {kwargs}") + + monkeypatch.setattr( + sglang_prepost_module, + "FunctionCallParser", + UnexpectedFunctionCallParser, + ) + + assert ( + build_tool_call_guided_decoding( + {"tool_choice": "auto", "tools": raw_tools}, + tool_call_parser_name="kimi_k2", + sglang_tools=convert_tools(raw_tools), + structural_tag_mode="off", + structural_tag_scope="always", + ) + is None + ) + + def test_builder_error_uses_required_json_fallback(self, monkeypatch): + raw_tools = [parity_tool()] + + class RaisingFunctionCallParser: + def __init__(self, **kwargs): + del kwargs + raise ValueError("unsupported schema") + + monkeypatch.setattr( + sglang_prepost_module, + "FunctionCallParser", + RaisingFunctionCallParser, + ) + + guided = build_tool_call_guided_decoding( + {"tool_choice": "required", "tools": raw_tools}, + tool_call_parser_name="kimi_k2", + sglang_tools=convert_tools(raw_tools), + structural_tag_mode="on", + structural_tag_scope="always", + ) + + assert guided is not None + assert set(guided) == {"json"} + def test_tool_parser_requires_tools(self): """Tool parser is not created if no tools in request.""" tcp, rp = create_parsers( @@ -2128,7 +2513,7 @@ def test_both_parsers(self): assert rp is not None def test_required_creates_json_array_parser(self): - """tool_choice='required' creates JsonArrayParser, not FunctionCallParser.""" + """Required choice without structural-tag guidance keeps JsonArrayParser.""" request = { "tools": [ { @@ -2148,7 +2533,7 @@ def test_required_creates_json_array_parser(self): assert isinstance(tcp, JsonArrayParser) def test_named_tool_choice_creates_json_array_parser(self): - """Named tool_choice creates JsonArrayParser.""" + """Named choice without structural-tag guidance keeps JsonArrayParser.""" request = { "tools": [ { @@ -3055,9 +3440,15 @@ class FakeTokenizer: exclude_tools_when_tool_choice_none=True, chat_template="custom template", default_thinking_mode="disabled", + structural_tag_mode="on", + structural_tag_scope="always", + structural_tag_schema="strict", ) assert sglang_processor_module._w_tokenizer.chat_template == "custom template" assert sglang_processor_module._w_default_thinking_mode == "disabled" + assert sglang_processor_module._w_structural_tag_mode == "on" + assert sglang_processor_module._w_structural_tag_scope == "always" + assert sglang_processor_module._w_structural_tag_schema == "strict" def test_with_reasoning_parser(self, tokenizer): """Reasoning parser is attached to result.""" diff --git a/components/src/dynamo/frontend/tests/test_structural_tag_policy.py b/components/src/dynamo/frontend/tests/test_structural_tag_policy.py new file mode 100644 index 000000000000..ce93dd3073c6 --- /dev/null +++ b/components/src/dynamo/frontend/tests/test_structural_tag_policy.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from dynamo.frontend.structural_tag_policy import ( + effective_tool_strict, + runtime_structural_tag_options, + should_attempt_structural_tag, +) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +@pytest.mark.parametrize( + ("mode", "scope", "choice", "strict", "single", "expected"), + [ + ("off", "always", "auto", False, False, False), + ("on", "always", "auto", False, False, True), + ("on", "auto", "auto", True, False, True), + ("on", "auto", "auto", False, True, True), + ("on", "auto", "auto", False, False, False), + ("on", "auto", "required", False, False, True), + ("on", "auto", "named", False, False, True), + ("on", "always", "none", True, True, False), + ], +) +def test_structural_tag_activation_policy( + mode, scope, choice, strict, single, expected +): + assert ( + should_attempt_structural_tag( + mode=mode, + scope=scope, + tool_choice_kind=choice, + has_tools=True, + any_explicit_strict=strict, + parallel_tool_calls_explicitly_false=single, + ) + is expected + ) + + +@pytest.mark.parametrize( + ("request_strict", "schema_mode", "expected"), + [ + (None, "auto", True), + (True, "auto", True), + (False, "auto", False), + (False, "strict", True), + ], +) +def test_effective_tool_strict(request_strict, schema_mode, expected): + assert effective_tool_strict(request_strict, schema_mode) is expected + + +def test_runtime_structural_tag_options_are_conservative_when_absent(): + assert runtime_structural_tag_options(None) == ("off", "auto", "auto") + assert runtime_structural_tag_options({}) == ("off", "auto", "auto") + + +def test_runtime_structural_tag_options_read_model_card_policy(): + assert runtime_structural_tag_options( + { + "structural_tag_mode": "on", + "structural_tag_scope": "always", + "structural_tag_schema": "strict", + } + ) == ("on", "always", "strict") diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index e48b956e55df..0989298a8f92 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -1397,6 +1397,8 @@ class RequestForSampling(SimpleNamespace): max_tokens=1, logprobs=True, top_logprobs=1, + top_k=None, + min_p=None, cache_salt=None, mm_processor_kwargs=None, ), @@ -1662,6 +1664,7 @@ class RequestForSampling(SimpleNamespace): max_tokens=1, cache_salt=None, mm_processor_kwargs=None, + logprobs=None, **{"temperature": None, "top_k": None, "min_p": None, **requested}, ) monkeypatch.setattr( @@ -2698,8 +2701,9 @@ def test_strict_schema_uses_copy(self, tokenizer): assert request.tools[0].function.strict is None assert parser.requests[0].messages is request.messages - # Auto schema mode must preserve an omitted strict flag as unset. - def test_auto_schema_preserves_unset_strict(self, tokenizer): + # In auto schema mode, omitted strict is schema-enforced. Only an explicit + # strict:false opts out. + def test_auto_schema_enforces_omitted_strict(self, tokenizer): parser = _FakeStructuralTagParser() guided = build_tool_call_guided_decoding( self._request(tokenizer), @@ -2709,7 +2713,203 @@ def test_auto_schema_preserves_unset_strict(self, tokenizer): structural_tag_schema="auto", ) - assert guided == {"structural_tag": {"format": {"strict": [None]}}} + assert guided == {"structural_tag": {"format": {"strict": [True]}}} + + def test_auto_schema_preserves_explicit_strict_false(self, tokenizer): + parser = _FakeStructuralTagParser() + request = self._request( + tokenizer, + tools=[ + { + **TOOL_REQUEST["tools"][0], + "function": { + **TOOL_REQUEST["tools"][0]["function"], + "strict": False, + }, + } + ], + ) + + guided = build_tool_call_guided_decoding( + request, + parser, + structural_tag_mode="on", + structural_tag_scope="always", + structural_tag_schema="auto", + ) + + assert guided == {"structural_tag": {"format": {"strict": [False]}}} + + @pytest.mark.parametrize( + "token_suffix", [":6124c78e", ""], ids=["suffixed", "plain"] + ) + @pytest.mark.parametrize("tool_choice_kind", ["auto", "required", "named"]) + @pytest.mark.parametrize("stream_response", [False, True], ids=["batch", "stream"]) + def test_hyv4_structural_tag_matches_checkpoint_tokens( + self, tokenizer, monkeypatch, token_suffix, tool_choice_kind, stream_response + ): + import vllm.envs as vllm_envs + import xgrammar as xgr + from vllm.tool_parsers.hy_v4_tool_parser import HYV4ToolParser + + # Dynamo's explicit policy must work without vLLM's environment gate. + monkeypatch.setattr(vllm_envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", False) + request = self._request( + tokenizer, + tools=[parity_tool()], + tool_choice=tool_choice_value(tool_choice_kind), + ) + markers = [ + f"<{closing}{name}{token_suffix}>" + for name in ("tool_calls", "tool_call", "arg_key", "arg_value") + for closing in ("", "/") + ] + # Only the vocabulary is needed to detect this checkpoint's suffix. + vocab = {marker: index for index, marker in enumerate(markers)} + parser = HYV4ToolParser( + SimpleNamespace(get_vocab=lambda: vocab, init_kwargs={}), request.tools + ) + assert parser.get_structural_tag(request) is None + + guided = build_tool_call_guided_decoding( + request, + parser, + structural_tag_mode="on", + structural_tag_scope="always", + ) + assert guided is not None + assert set(guided) == {"structural_tag"} + compiler = xgr.GrammarCompiler( + xgr.TokenizerInfo(["x"], vocab_type=xgr.VocabType.RAW), max_threads=1 + ) + compiled = compiler.compile_structural_tag(json.dumps(guided["structural_tag"])) + native_output = ( + f"get_weather" + f"city" + f"Paris" + f"" + ) + assert xgr.GrammarMatcher(compiled).accept_string(native_output) + assert not xgr.GrammarMatcher(compiled).accept_string( + native_output.replace("get_weather", "unknown_tool") + ) + + post = StreamingPostProcessor( + tokenizer=tokenizer, + request_for_sampling=request, + sampling_params=SamplingParams(), + prompt_token_ids=[], + tool_parser=parser, + reasoning_parser_class=None, + chat_template_kwargs={}, + stream_response=stream_response, + ) + choice = post.process_output( + SimpleNamespace( + index=0, + text=native_output, + token_ids=list(vocab.values()), + finish_reason="stop", + logprobs=None, + ) + ) + assert choice is not None + assert choice["finish_reason"] == "tool_calls" + assert not choice["delta"].get("content") + function = choice["delta"]["tool_calls"][0]["function"] + assert function["name"] == "get_weather" + assert json.loads(function["arguments"]) == {"city": "Paris"} + + def test_structural_tag_builder_error_uses_forced_json_fallback(self, tokenizer): + class RaisingParser(_FakeStructuralTagParser): + def get_structural_tag(self, request, *, reasoning=False): + del request, reasoning + raise ValueError("unsupported schema") + + guided = build_tool_call_guided_decoding( + self._request(tokenizer, tool_choice="required"), + RaisingParser(), + structural_tag_mode="on", + structural_tag_scope="always", + ) + + assert guided is not None + assert set(guided) == {"json"} + + @pytest.mark.parametrize("strict, expects_tag", [(True, True), (False, False)]) + def test_declared_model_registry_matches_pinned_vllm_auto_floor( + self, tokenizer, monkeypatch, strict, expects_tag + ): + seen = {} + + class RegistryParser: + structural_tag_model = "qwen_3_coder" + + def get_structural_tag(self, request): + raise AssertionError(f"unexpected parser-level env gate: {request}") + + # vLLM 0.29.0 has no strict_level argument and returns no auto tag + # when every tool is explicitly non-strict. + def fake_get_model_structural_tag(*, model, tools, tool_choice, reasoning): + seen.update( + model=model, + strict=tools[0].function.strict, + tool_choice=tool_choice, + reasoning=reasoning, + ) + return _FakeStructuralTag({"format": {"type": "tag"}}) if strict else None + + monkeypatch.setattr( + prepost_module, + "get_model_structural_tag", + fake_get_model_structural_tag, + ) + + request = self._request( + tokenizer, + tools=[ + { + **TOOL_REQUEST["tools"][0], + "function": { + **TOOL_REQUEST["tools"][0]["function"], + "strict": strict, + }, + } + ], + ) + guided = build_tool_call_guided_decoding( + request, + RegistryParser(), + structural_tag_mode="on", + structural_tag_scope="always", + ) + + assert guided == ( + {"structural_tag": {"format": {"type": "tag"}}} if expects_tag else None + ) + assert seen == { + "model": "qwen_3_coder", + "strict": strict, + "tool_choice": "auto", + "reasoning": False, + } + + def test_missing_registry_uses_parser_method(self, tokenizer, monkeypatch): + class CompatibleParser(_FakeStructuralTagParser): + structural_tag_model = "qwen_3_coder" + + parser = CompatibleParser() + monkeypatch.setattr(prepost_module, "get_model_structural_tag", None) + + guided = build_tool_call_guided_decoding( + self._request(tokenizer), + parser, + structural_tag_mode="on", + structural_tag_scope="always", + ) + + assert guided == {"structural_tag": {"format": {"strict": [True]}}} + assert len(parser.requests) == 1 # Parser-created grammar must survive preprocessing as guided decoding. @pytest.mark.asyncio @@ -3410,22 +3610,6 @@ def test_runtime_config_context_length(vllm_processor_module, runtime_config, ex assert vllm_processor_module._runtime_config_context_length(mdc) == expected -def test_runtime_config_structural_tag_options(vllm_processor_module): - mdc = SimpleNamespace( - runtime_config=lambda: { - "structural_tag_mode": "on", - "structural_tag_scope": "always", - "structural_tag_schema": "strict", - } - ) - - assert vllm_processor_module._runtime_config_structural_tag_options(mdc) == ( - "on", - "always", - "strict", - ) - - # Regression: MistralTokenizer (--tokenizer-mode mistral) has no chat_template # attribute; a direct read used to crash vLLM preprocessing. diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index deb83e92361c..64514d486d1f 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -45,6 +45,7 @@ from dynamo.vllm.errors import vllm_client_error_to_http_error from .prepost import StreamingPostProcessor, preprocess_chat_request +from .structural_tag_policy import runtime_structural_tag_options from .thinking import runtime_default_thinking_mode from .utils import ( as_error_envelope, @@ -414,19 +415,6 @@ def _runtime_config_context_length(mdc: ModelDeploymentCard) -> int | None: return context_length -def _runtime_config_structural_tag_options( - mdc: ModelDeploymentCard, -) -> tuple[str, str, str]: - runtime_config = mdc.runtime_config() - if not isinstance(runtime_config, dict): - return "off", "auto", "auto" - return ( - runtime_config.get("structural_tag_mode", "off"), - runtime_config.get("structural_tag_scope", "auto"), - runtime_config.get("structural_tag_schema", "auto"), - ) - - def _ensure_chat_template( tokenizer: Any, local_dir: str, chat_template_flag: str | None ) -> None: @@ -1604,7 +1592,7 @@ async def chat_engine_factory( structural_tag_mode, structural_tag_scope, structural_tag_schema, - ) = _runtime_config_structural_tag_options(mdc) + ) = runtime_structural_tag_options(mdc.runtime_config()) block_size = self.config.kv_cache_block_size or 16 diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index eeb05cdcac20..79e15b7aafd6 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -8,6 +8,7 @@ from typing import Any, List, Optional import sglang as sgl +from sglang.srt.parser.reasoning_parser import ReasoningParser from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -31,6 +32,7 @@ register_model, ) from dynamo.sglang._compat import ( + filter_supported_async_generate_kwargs, sglang_uses_mla_backend, supports_disagg_prefill_cancel_anytime, ) @@ -57,6 +59,44 @@ SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake" SPEC_DECODE_RUNTIME_KEY = "spec_decode" +TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY = ( + "tool_call_structural_tag_reasoning_gate" +) + + +def publish_sglang_structural_tag_reasoning_policy( + runtime_config: ModelRuntimeConfig, + engine: Optional[sgl.Engine], + server_args: ServerArgs, +) -> None: + """Advertise the native gate that consumes reasoning before guided output.""" + # vLLM also publishes grammar timing through runtime metadata. + # SGLang's exclusion additionally depends on require_reasoning per request. + parser = getattr(server_args, "reasoning_parser", None) + if ( + engine is None + or not parser + or getattr(server_args, "skip_tokenizer_init", False) + ): + return + tokenizer = getattr(engine.tokenizer_manager, "tokenizer", None) + if ( + tokenizer is None + or "require_reasoning" + not in filter_supported_async_generate_kwargs( + engine, {"require_reasoning": True} + ) + ): + return + # Match the scheduler's gate initialization, including disabled gates when + # the reasoning terminator cannot be encoded by the initialized tokenizer. + reasoner = ReasoningParser( + model_type=parser, stream_reasoning=False, tokenizer=tokenizer + ) + if tokenizer.encode(reasoner.detector.think_end_token, add_special_tokens=False): + runtime_config.set_engine_specific( + TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, json.dumps(True) + ) def _supports_engine_generate( @@ -447,6 +487,11 @@ async def get_runtime_config( # set reasoning parser and tool call parser runtime_config.reasoning_parser = dynamo_args.dyn_reasoning_parser runtime_config.tool_call_parser = dynamo_args.dyn_tool_call_parser + # Multimodal and diffusion handlers do not forward the per-request gate. + if llm_handler and not getattr(server_args, "dllm_algorithm", None): + publish_sglang_structural_tag_reasoning_policy( + runtime_config, engine, server_args + ) if dynamo_args.dyn_default_thinking_mode is not None: runtime_config.set_engine_specific( "default_thinking_mode", diff --git a/components/src/dynamo/sglang/tests/test_runtime_metadata.py b/components/src/dynamo/sglang/tests/test_runtime_metadata.py index 0d1b94cbc181..110636cb4767 100644 --- a/components/src/dynamo/sglang/tests/test_runtime_metadata.py +++ b/components/src/dynamo/sglang/tests/test_runtime_metadata.py @@ -4,6 +4,7 @@ import asyncio import json from types import SimpleNamespace +from unittest.mock import Mock import pytest @@ -23,6 +24,62 @@ ] +@pytest.mark.parametrize( + "parser, skip_tokenizer_init, has_engine, has_tokenizer, supports_gate, think_end_ids, expected", + [ + ("kimi_k3", False, True, True, True, [163588, 39964, 163589], True), + (None, False, True, True, True, [163588], False), + ("kimi_k3", True, True, True, True, [163588], False), + ("kimi_k3", False, False, True, True, [163588], False), + ("kimi_k3", False, True, False, True, [163588], False), + ("kimi_k3", False, True, True, False, [163588], False), + ("kimi_k3", False, True, True, True, [], False), + ], +) +def test_structural_tag_reasoning_policy_advertises_only_an_available_gate( + parser, + skip_tokenizer_init, + has_engine, + has_tokenizer, + supports_gate, + think_end_ids, + expected, +): + from dynamo.sglang import register + + async def async_generate(*, require_reasoning=False): + pass + + async def legacy_async_generate(*, sampling_params=None): + pass + + tokenizer = Mock() + tokenizer.encode.return_value = think_end_ids + engine = ( + SimpleNamespace( + tokenizer_manager=SimpleNamespace( + tokenizer=tokenizer if has_tokenizer else None + ), + async_generate=async_generate if supports_gate else legacy_async_generate, + ) + if has_engine + else None + ) + server_args = SimpleNamespace( + reasoning_parser=parser, skip_tokenizer_init=skip_tokenizer_init + ) + runtime_config = register.ModelRuntimeConfig() + + register.publish_sglang_structural_tag_reasoning_policy( + runtime_config, engine, server_args + ) + + key = register.TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY + assert (key in runtime_config.runtime_data) is expected + if expected: + assert json.loads(runtime_config.runtime_data[key]) is True + + @pytest.mark.parametrize( "server_args, expected", [ @@ -162,6 +219,8 @@ def test_runtime_config_publishes_supported_disagg_capabilities( disaggregation_mode=disaggregation_mode, max_prefill_tokens=None, page_size=16, + reasoning_parser="kimi_k3", + skip_tokenizer_init=False, speculative_algorithm="NONE", speculative_num_steps=None, ) @@ -188,8 +247,16 @@ def test_runtime_config_publishes_supported_disagg_capabilities( ) engine = None if runtime_supported: + + async def async_generate(*, require_reasoning=False): + pass + + tokenizer = Mock() + tokenizer.encode.return_value = [163588, 39964, 163589] engine = SimpleNamespace( + async_generate=async_generate, tokenizer_manager=SimpleNamespace( + tokenizer=tokenizer, context_len=4096, validate_total_tokens=True, num_reserved_tokens=0, @@ -209,6 +276,10 @@ def test_runtime_config_publishes_supported_disagg_capabilities( assert (capability in runtime_config.runtime_data) is capability_expected if capability_expected: assert json.loads(runtime_config.runtime_data[capability]) is True + reasoning_gate = register.TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY + assert (reasoning_gate in runtime_config.runtime_data) is ( + runtime_supported and not enable_multimodal + ) assert "Failed to get runtime config" not in caplog.text diff --git a/docs/fern/pages/reference/components/runtime-configuration.mdx b/docs/fern/pages/reference/components/runtime-configuration.mdx index 084e7eae48f7..fb7004ee22a0 100644 --- a/docs/fern/pages/reference/components/runtime-configuration.mdx +++ b/docs/fern/pages/reference/components/runtime-configuration.mdx @@ -296,14 +296,14 @@ address. Environment variable: `DYN_EXCLUDE_TOOLS_WHEN_TOOL_CHOICE_NONE` - - Enable structural tag guided decoding for tool calls. When enabled, configure activation scope and parameter schema strictness with `--dyn-structural-tag-scope` and `--dyn-structural-tag-schema`. + + Enable structural tag guided decoding for tool calls when the configured parser and backend support it. The Rust, Python vLLM, and Python SGLang frontend preprocessing paths use this setting for `auto`, required, and named tool choices. Setting it to `false` disables optional guidance. Rust preserves native tags for Kimi K2 required/named and Kimi K3 named choices; Python vLLM and SGLang respect the opt-out for all tool choices. Native Rust sidecars retain their conservative `off`/`auto` behavior, including this Kimi exception, and are not included in the default change. Environment variable: `DYN_ENABLE_STRUCTURAL_TAG` - - Controls when structural tags are activated. `auto` activates them for required or named `tool_choice`, or when any tool has `strict=true` or `parallel_tool_calls` is false. `always` additionally activates them for `tool_choice=auto` without those conditions. `tool_choice=none` is unaffected by either setting. Only meaningful when `--dyn-enable-structural-tag` is set. + + Controls when structural tags are activated. `always` activates them for supported `auto`, required, and named tool requests. `auto` preserves the legacy conditional behavior: required or named `tool_choice`, or automatic tool choice when any tool has `strict=true` or `parallel_tool_calls` is false. `tool_choice=none` is unaffected by either setting. Applies when structural tags are active. Allowed values: auto always @@ -311,7 +311,7 @@ address. - Controls parameter schema strictness inside structural tags. `auto` applies the real parameter schema unless the tool explicitly sets `strict=false`. Tools with `strict=false` remain syntactically constrained but schema-unconstrained. `strict` applies the real parameter schema to every tool. Only meaningful when `--dyn-enable-structural-tag` is set. + Controls parameter schema strictness inside structural tags. `auto` applies the declared parameter schema unless a tool explicitly sets `strict=false`; an omitted `strict` field is schema-enforced. `strict` applies the declared schema to every tool and overrides `strict=false`. Applies when structural tags are active. Allowed values: auto strict diff --git a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md index c23f91afc6d1..a54b02bde23e 100644 --- a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md +++ b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md @@ -1,5 +1,5 @@ --- -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: Structural Tag (Guided Decoding for Tool Calls) subtitle: Constrain model output to valid tool call format using xgrammar structural tags @@ -25,13 +25,22 @@ Benefits: ## Prerequisites -- A backend engine with xgrammar support. +- A backend engine that accepts structural-tag/xgrammar guided decoding. - A Dynamo tool call parser that provides a structural tag config (see [Supported Parsers](#supported-parsers) below). ## Quick Start -Enable structural tags on the **worker** with `--dyn-enable-structural-tag`, alongside the tool-call parser. The Frontend needs no extra flags: +Structural tags are enabled by default. Configure the tool-call parser on the +**worker**; the Frontend needs no extra flags: + +> [!NOTE] +> TensorRT-LLM guided decoding remains **opt-in** pending +> [TensorRT-LLM #19913](https://github.com/NVIDIA/TensorRT-LLM/issues/19913). +> Its `--guided-decoding-backend` default is unset, so default deployments ignore +> structural tags and do not enforce their tool argument schemas. For affected +> deployments using `--guided-decoding-backend xgrammar`, pass +> `--no-dyn-enable-structural-tag` until a supported upgrade resolves the issue. ```yaml apiVersion: nvidia.com/v1beta1 @@ -70,7 +79,6 @@ spec: - Qwen/Qwen3.5-4B - --dyn-tool-call-parser - qwen3_coder - - --dyn-enable-structural-tag ``` Eligible tool-calling requests will now use xgrammar structural tags for guided @@ -80,45 +88,100 @@ decoding. See [Activation Scope](#activation-scope) for the exact policy. | Flag | Values | Default | Description | |---|---|---|---| -| `--dyn-enable-structural-tag` | bool | `false` | Master switch. When disabled, tool calling works the same as without structural tags. | -| `--dyn-structural-tag-scope` | `auto`, `always` | `auto` | Controls when structural tags are activated (see [Activation Scope](#activation-scope)). | +| `--dyn-enable-structural-tag` | bool | `true` | Enables tool-call structural tags in Rust, Python vLLM, and Python SGLang preprocessing. Rust retains native forced-tool tags for Kimi K2 required/named and Kimi K3 named choices when disabled. | +| `--dyn-structural-tag-scope` | `auto`, `always` | `always` | Controls when structural tags are activated (see [Activation Scope](#activation-scope)). | | `--dyn-structural-tag-schema` | `auto`, `strict` | `auto` | Controls parameter schema strictness inside structural tags (see [Schema Modes](#schema-modes)). | +## Preserve Previous Behavior + +To disable optional structural guidance, pass the negated flag to the worker: + +```bash +python3 -m dynamo.vllm ... --no-dyn-enable-structural-tag +python3 -m dynamo.sglang ... --no-dyn-enable-structural-tag +python3 -m dynamo.trtllm ... --no-dyn-enable-structural-tag +``` + +Or set the equivalent environment variable before starting the worker: + +```bash +export DYN_ENABLE_STRUCTURAL_TAG=false +``` + +Rust preprocessing still applies native tags to Kimi K2 `required`/named +choices and Kimi K3 named choices when structural tags are disabled. Kimi K3 +named tags use the new argument-schema enforcement; the opt-out does not +restore main's grammar. A lookahead pattern can still be rejected by XGrammar +0.2.1 or 0.2.7 unless the tool sets `strict: false` under schema mode `auto`. +Global schema mode `strict` overrides that workaround. Kimi K2/K3 automatic +choices and Kimi K3 `required` do not use this exception. Python vLLM and SGLang +preprocessing respect the opt-out for all tool choices. + +To keep structural tags enabled but preserve the previous conditional activation +policy, set the scope to `auto`: + +```bash +export DYN_STRUCTURAL_TAG_SCOPE=auto +``` + +With this scope, required and named tool choices remain eligible. Automatic tool +choice uses structural tags only when a tool sets `strict: true` or the request +sets `parallel_tool_calls` to `false`. + +Setting `strict: false` on a tool relaxes its argument schema. On vLLM +0.30.0, pinned by Dynamo's CUDA image, auto choice with every tool explicitly +non-strict also gets no structural tag from the registry; see +[Schema Modes](#schema-modes). Use the global opt-out above to disable optional +structural guidance. + ## Supported Parsers -Not all parsers support structural tags. Parsers without a structural tag -config fall back to standard behaviour (a warning is logged if structural -tags are enabled but the parser does not support them). +Not all parsers support structural tags. Parsers without a structural-tag +builder fall back to their existing best-effort tool-calling behavior without +rejecting the request. Currently tested and supported: - `qwen3_coder`, `nemotron_nano` - `hermes`, `qwen25` - `deepseek_v3_2`, `deepseek_v4` +- `kimi_k2`, `kimi_k3`, `kimi-k3` +- `inkling` Contributions adding structural tag support for new parsers are welcome. +This list describes Dynamo's Rust parser registry. The Python vLLM and SGLang +frontend processors apply the same mode, scope, and schema policy through the +tool parser supplied by their installed engine version. Parser availability can +therefore differ by backend and engine version. + +> [!NOTE] +> Native Rust sidecars retain their existing conservative `off`/`auto` +> settings, including the native Kimi forced-tool exception described above, +> and are not included in this default change. The default-on policy +> applies when regular vLLM, SGLang, or TensorRT-LLM workers publish the +> deployment runtime configuration consumed by frontend preprocessing. + ## Activation Scope The `--dyn-structural-tag-scope` flag controls when structural tags are used based on the request's `tool_choice`: -### `auto` (default) +### `always` (default) | `tool_choice` | Structural tag? | |---|---| | `required` / `named` | Always | -| `auto` | Only when any tool has `strict: true` or `parallel_tool_calls` is `false` | -| `none` | Exclusion tag only (bans tool call tokens, see [below](#tool_choicenone-and-token-banning)) | +| `auto` | Always attempted; the parser may return no tag | +| `none` | Exclusion tag on the Rust path only | -### `always` +### `auto` (legacy conditional activation) | `tool_choice` | Structural tag? | |---|---| | `required` / `named` | Always | -| `auto` | Always | -| `none` | Exclusion tag only | - +| `auto` | Only when any tool has `strict: true` or `parallel_tool_calls` is `false` | +| `none` | Exclusion tag on the Rust path only (see [below](#tool_choicenone-and-token-banning)) | ## Schema Modes @@ -127,19 +190,45 @@ tool arguments inside the structural tag: ### `auto` (default) -- Tools with `strict: true` or without `strict` — their actual parameter schema is used. -- Tools with `strict: false` — an unconstrained schema is used, allowing the model to generate any valid content in the parser's native format. +- Tools with omitted `strict` or `strict: true` — their declared parameter + schema is used. +- Tools with `strict: false` — argument content is schema-relaxed when the + parser builds a structural tag. ### `strict` - All tools use their actual parameter schema regardless of the `strict` flag. +If a model-native builder cannot safely represent part of a schema, it keeps +the strongest safe tool envelope and relaxes that argument section. If the +builder cannot produce a structural tag at all, Dynamo uses the existing +compatibility path rather than introducing a new request error; automatic tool +choice may therefore remain unconstrained for that parser/schema combination. + +> [!WARNING] +> Valid tool schemas containing constructs unsupported by the backend's +> XGrammar version, such as regex lookahead, can cause request rejection. +> Backend compilation errors do not trigger Dynamo's builder fallback. +> With XGrammar 0.2.1 or 0.2.7 and schema mode `auto`, setting the tool +> function's `strict: false` avoids this lookahead rejection by disabling +> argument-schema enforcement. This workaround is not guaranteed on older +> versions. The deployment opt-out still retains Rust Kimi K3 named-call tags. + +In vLLM 0.30.0, pinned by Dynamo's CUDA image, `tool_choice="auto"` returns no +structural tag when every tool is explicitly `strict: false`. The `always` +activation scope still attempts the tag, but cannot enforce the native tool +envelope for that request. Set schema mode to `strict` to override the opt-out +and let vLLM build the tag. + ## `tool_choice="none"` and Token Banning -When `tool_choice="none"` and structural tags are enabled, Dynamo injects an -exclusion structural tag that bans parser-specific tool-call start tokens (for -example ``) so the model cannot start native tool-call syntax. +On the Rust frontend preprocessing path, when `tool_choice="none"` and +structural tags are enabled, Dynamo injects an exclusion structural tag that +bans parser-specific tool-call start tokens (for example ``) so the +model cannot start native tool-call syntax. The Python vLLM and SGLang frontend +processors continue to handle `none` through their existing prompt and response +shaping; this release does not add token banning to those paths. **Quality trade-off**. If tools remain in the prompt on `none` (often via `--no-exclude-tools-when-tool-choice-none` to keep the chat prefix stable for KV @@ -168,7 +257,8 @@ KV cache reuse. ## Example -To pin the scope and schema, add `--dyn-structural-tag-scope` and `--dyn-structural-tag-schema` to the worker `args:` alongside the parser and master switch: +To pin the enabled state, scope, and schema explicitly, add the following flags +to the worker `args:` alongside the parser: ```yaml - name: SGLangWorker diff --git a/docs/fern/pages/use-cases/tool-calling-and-reasoning/tool-call-parsing.mdx b/docs/fern/pages/use-cases/tool-calling-and-reasoning/tool-call-parsing.mdx index 289b03f206b1..a0a61da9ff49 100644 --- a/docs/fern/pages/use-cases/tool-calling-and-reasoning/tool-call-parsing.mdx +++ b/docs/fern/pages/use-cases/tool-calling-and-reasoning/tool-call-parsing.mdx @@ -350,6 +350,6 @@ The **Upstream name** column shows where the vLLM or SGLang parser name differs - -You can enable **xgrammar structural tags** so guided decoding matches the parser's tool-call format at token granularity. See [Structural Tag (Guided Decoding for Tool Calls)](structural-tag.md). + +For parser and backend combinations that support them, Dynamo enables **xgrammar structural tags** by default so guided decoding matches the parser's tool-call format at token granularity. See [Structural Tag (Guided Decoding for Tool Calls)](structural-tag.md) for the activation policy, compatibility fallbacks, and opt-out settings. diff --git a/docs/fern/translations/zh-CN/pages/use-cases/tool-calling-and-reasoning/tool-call-parsing.mdx b/docs/fern/translations/zh-CN/pages/use-cases/tool-calling-and-reasoning/tool-call-parsing.mdx index 2349e18a83be..6b73ec4fb4e9 100644 --- a/docs/fern/translations/zh-CN/pages/use-cases/tool-calling-and-reasoning/tool-call-parsing.mdx +++ b/docs/fern/translations/zh-CN/pages/use-cases/tool-calling-and-reasoning/tool-call-parsing.mdx @@ -105,6 +105,6 @@ Dynamo 会从模型输出中解析出工具调用，并在响应中以兼容 Ope > 如果工具调用返回结果不正确，请向单个复现请求添加 `"logprobs": true` 并分享响应。有关报告问题时需要捕获和包含的内容，请参阅 > [工具调用故障排查](../../../../../pages/use-cases/tool-calling-and-reasoning/troubleshooting-tool-calls.md)。 -## 可选：结构化标签（structural tags） +## 默认启用结构化标签（structural tags） -你可以启用 **xgrammar 结构化标签**，让引导式解码在 token 粒度上匹配解析器的工具调用格式。参阅 [Structural tag (guided decoding for tool calls)](../../../../../pages/use-cases/tool-calling-and-reasoning/structural-tag.md)。 +对于支持结构化标签的解析器和后端组合，Dynamo 默认启用 **xgrammar 结构化标签**，使引导式解码在 token 粒度上匹配解析器的工具调用格式。有关激活策略、兼容性回退和退出设置，请参阅 [Structural tag (guided decoding for tool calls)](../../../../../pages/use-cases/tool-calling-and-reasoning/structural-tag.md)。 diff --git a/lib/backend-common/src/worker.rs b/lib/backend-common/src/worker.rs index aadc3d8b2df3..4290689f907d 100644 --- a/lib/backend-common/src/worker.rs +++ b/lib/backend-common/src/worker.rs @@ -264,6 +264,9 @@ impl Default for WorkerConfig { metrics_labels: Vec::new(), disaggregation_mode: DisaggregationMode::Aggregated, health_check_payload: None, + // Keep the shared/wire default conservative for mixed-version and + // Rust-sidecar compatibility. Deployment-facing runtime arguments + // explicitly publish the current On/Always defaults. structural_tag_mode: StructuralTagMode::Off, structural_tag_scope: StructuralTagScope::Auto, structural_tag_schema: StructuralTagSchemaMode::Auto, diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index 79f0d0d2da39..e40a567e8f74 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -2295,9 +2295,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.6" +version = "9.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" +checksum = "a751a31502748c656f69bd1fab01f1d844b6bb629b69e9fbed0b0da9af916b7a" dependencies = [ "aho-corasick", "anyhow", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index edd5ea7f4a07..30e520944bc1 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2347,9 +2347,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.6" +version = "9.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" +checksum = "a751a31502748c656f69bd1fab01f1d844b6bb629b69e9fbed0b0da9af916b7a" dependencies = [ "aho-corasick", "anyhow", diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index 870125506767..bea94cea557e 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -62,7 +62,7 @@ request-trace-s3 = ["dynamo-llm/request-trace-s3"] aisimulate-core = { version = "=0.13.0-dev.202610060000000067", optional = true, features = ["python"] } dynamo-runtime = { path = "../../runtime" } dynamo-mocker = { path = "../../mocker" } -dynamo-parsers = { version = "9.2.6" } +dynamo-parsers = { version = "=9.2.7" } dynamo-backend-common = { path = "../../backend-common" } dynamo-sidecar-common = { path = "../../sidecar/common" } dynamo-sglang-sidecar = { path = "../../sidecar/sglang" } diff --git a/lib/llm/src/http/service.rs b/lib/llm/src/http/service.rs index 3094b5840157..255c1709800b 100644 --- a/lib/llm/src/http/service.rs +++ b/lib/llm/src/http/service.rs @@ -361,7 +361,7 @@ mod tests { assert_eq!( result.guided_tool_constraint, GuidedToolConstraint::StructuralTag, - "kimi_k2 + required must use the intrinsic structural tag, not a reconstructed JSON schema" + "kimi_k2 + required must retain its native structural tag when mode is off" ); } @@ -377,7 +377,7 @@ mod tests { assert_eq!( result.guided_tool_constraint, GuidedToolConstraint::StructuralTag, - "kimi_k2 + a named tool choice must use the intrinsic structural tag, not a reconstructed JSON schema" + "kimi_k2 + a named tool choice must retain its native structural tag when mode is off" ); } @@ -423,11 +423,12 @@ mod tests { ); } - // With the operator default (`structural_tag_mode = Off`), the same non-Kimi - // parser must NOT get a structural tag — confirms the mode gate above is real, - // not a permanently-on regression. + // With the low-level ParsingOptions default (`structural_tag_mode = Off`), + // the same non-Kimi parser must NOT get a structural tag. Regular workers + // explicitly publish the deployment's default-on policy into the model card; + // native sidecars that do not do so retain this conservative default. #[test] - fn operator_default_mode_off_does_not_resolve_structural_tag_for_a_non_kimi_parser() { + fn parsing_options_default_mode_off_does_not_resolve_structural_tag() { let parsing_options = ParsingOptions { tool_call_parser: Some("qwen3_coder".to_string()), ..Default::default() diff --git a/lib/llm/src/local_model/runtime_config.rs b/lib/llm/src/local_model/runtime_config.rs index 7fd56f49bd73..3008f6f58099 100644 --- a/lib/llm/src/local_model/runtime_config.rs +++ b/lib/llm/src/local_model/runtime_config.rs @@ -42,6 +42,11 @@ pub const TOKEN_BUDGET_RUNTIME_KEY: &str = "token_budget"; pub const TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY: &str = "tool_call_structural_tag_excludes_reasoning"; +/// A backend consumes the reasoning prefix only when `require_reasoning` is true +/// on the request. Missing metadata leaves reasoning in the frontend grammar. +pub const TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY: &str = + "tool_call_structural_tag_reasoning_gate"; + /// Describes which request-token overflows the frontend may reject early. /// /// The combined limit already accounts for engine-reserved tokens. A false @@ -391,6 +396,8 @@ impl Default for ModelRuntimeConfig { tool_call_arguments_format: ToolCallArgumentsFormat::JsonString, tokenizer_backend: None, tokenizer_fallback_enabled: None, + // Missing fields from older workers remain conservative. Current + // deployment configuration explicitly publishes On/Always. structural_tag_mode: StructuralTagMode::Off, structural_tag_scope: StructuralTagScope::Auto, structural_tag_schema: StructuralTagSchemaMode::Auto, diff --git a/lib/llm/src/preprocessor/structural_tag.rs b/lib/llm/src/preprocessor/structural_tag.rs index c3431be57de0..45bca09450c6 100644 --- a/lib/llm/src/preprocessor/structural_tag.rs +++ b/lib/llm/src/preprocessor/structural_tag.rs @@ -5,6 +5,7 @@ use crate::local_model::runtime_config::{ StructuralTagMode, StructuralTagScope, TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, }; use crate::preprocessor::{OpenAIPreprocessor, PreprocessedRequest}; use crate::protocols::openai::tools::{ToolChoiceValidation, validate_tool_choice_against_names}; @@ -46,16 +47,26 @@ fn is_kimi_k2_parser(parser_name: Option<&str>) -> bool { } fn requires_intrinsic_structural_tag(parser_name: Option<&str>, tool_choice: &ToolChoice) -> bool { - // K2 forced calls and K3 named calls cannot use Dynamo's generic JSON-schema - // fallback because both families emit native, marker-delimited formats. - // Treat their structural tags as part of implementing these standard OpenAI - // request shapes, not as an operator opt-in. K3 required remains on its - // intentional prompt-level XTML path. + // Preserve native forced-tool grammars even when optional guidance is off. + // K3 required retains its existing prompt/parser path. (is_kimi_k2_parser(parser_name) && matches!(tool_choice, ToolChoice::Required | ToolChoice::Named(_))) || (is_kimi_k3_parser(parser_name) && matches!(tool_choice, ToolChoice::Named(_))) } +pub(super) fn requires_native_tool_call_format( + parser_name: Option<&str>, + tool_choice: &ToolChoice, +) -> bool { + // The generic forced-tool JSON grammar describes a different wire format + // from Kimi's native marker-delimited calls. This also covers K3 required, + // which uses its prompt/parser path when structural guidance is off. + (is_kimi_k2_parser(parser_name) + && matches!(tool_choice, ToolChoice::Required | ToolChoice::Named(_))) + || (is_kimi_k3_parser(parser_name) + && matches!(tool_choice, ToolChoice::Required | ToolChoice::Named(_))) +} + fn should_skip_tool_call_ban(exclude_tools_when_none: bool, tool_choice: &ToolChoice) -> bool { exclude_tools_when_none && matches!(tool_choice, ToolChoice::None) } @@ -64,9 +75,9 @@ fn should_skip_tool_call_ban(exclude_tools_when_none: bool, tool_choice: &ToolCh /// parser registry can actually build one. /// /// This is the single owner of "is a structural tag applicable and available" — -/// covering model-family/tool-choice intrinsic eligibility, the operator's global -/// `structural_tag_mode`/`structural_tag_scope`, the `tool_choice=None` ban-tag -/// exclusion, and real parser-registry builder availability. A caller can only +/// covering the operator's global `structural_tag_mode`/`structural_tag_scope`, +/// intrinsic Kimi forced-tool grammars, the `tool_choice=None` ban-tag exclusion, and real +/// parser-registry builder availability. A caller can only /// reach `Required` by way of a real, registered /// [`dynamo_parsers::tool_calling::StructuralTagBuilder`] — it cannot ask for the /// eligibility half of this decision without also getting the registry-availability @@ -96,7 +107,6 @@ impl StructuralTagDecision { /// (`http::service::apply_request_tool_call_parsing_options`) must propagate /// this error rather than falling back to `NotApplicable`, or an invalid forced /// choice would silently install a structural tag anyway. -#[allow(clippy::too_many_arguments)] pub(crate) fn structural_tag_decision( parser_name: Option<&str>, tool_choice: &ToolChoice, @@ -106,7 +116,7 @@ pub(crate) fn structural_tag_decision( scope: StructuralTagScope, exclude_tools_when_tool_choice_none: bool, ) -> Result { - // Validate before any mode or family gate. Kimi K3 required requests use a + // Validate before any policy gate. Kimi K3 required requests use a // prompt-level XTML path, but they still need an actual tool to require. validate_forced_tool_choice(tool_choice, tools)?; @@ -124,7 +134,7 @@ pub(crate) fn structural_tag_decision( } let Some(parser_name) = parser_name else { - tracing::warn!( + tracing::debug!( "Structural tag is enabled but --dyn-tool-call-parser is not set; \ structural tags will not be applied" ); @@ -194,23 +204,34 @@ impl OpenAIPreprocessor { parallel_tool_calls, schema_mode: self.runtime_config.structural_tag_schema, starts_in_reasoning: prompt_injected_reasoning - && !self.tool_call_structural_tag_excludes_reasoning(), + && !self.tool_call_structural_tag_excludes_reasoning( + preprocessed_request.require_reasoning, + ), }; Self::apply_tool_call_format(parser_name, builder, &ctx, preprocessed_request) } - fn tool_call_structural_tag_excludes_reasoning(&self) -> bool { - match self - .runtime_config - .get_engine_specific::(TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY) - { + fn tool_call_structural_tag_excludes_reasoning(&self, require_reasoning: bool) -> bool { + // vLLM advertises exclusion for the deployment; SGLang gates it per request. + // Once the backend consumes THINK_CLOSE, this grammar must not expect it again. + // Auto tool choice alone does not activate SGLang's gate. + self.structural_tag_reasoning_metadata( + TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + ) || (require_reasoning + && self.structural_tag_reasoning_metadata( + TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, + )) + } + + fn structural_tag_reasoning_metadata(&self, key: &str) -> bool { + match self.runtime_config.get_engine_specific::(key) { Ok(Some(excludes_reasoning)) => excludes_reasoning, Ok(None) => false, Err(error) => { tracing::warn!( %error, - key = TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + key, "Ignoring invalid structural-tag reasoning metadata; using the compatibility behavior" ); false @@ -228,7 +249,7 @@ impl OpenAIPreprocessor { .and_then(|tc| tc.structural_tag_builder.as_ref()); if builder.is_none() { - tracing::warn!( + tracing::debug!( parser = parser_name, "Structural tag enabled but parser does not support it; \ falling back to default behaviour" @@ -278,12 +299,12 @@ impl OpenAIPreprocessor { return Ok(false); } Err(e) => { - return Err(DynamoError::builder() - .error_type(ErrorType::Unknown) - .message(format!( - "failed to build structural_tag for parser '{parser_name}': {e}" - )) - .build()); + tracing::warn!( + parser = parser_name, + error = %e, + "Failed to build structural tag; using compatibility fallback" + ); + return Ok(false); } }; @@ -425,78 +446,103 @@ mod tests { } #[test] - fn named_kimi_k3_is_intrinsic_even_when_global_mode_is_off() { - let named = ToolChoice::Named("get_weather".to_string()); - assert!(requires_intrinsic_structural_tag(Some("kimi_k3"), &named)); - assert!(requires_intrinsic_structural_tag(Some("kimi-k3"), &named)); - } - - #[test] - fn forced_kimi_k2_is_intrinsic_even_when_global_mode_is_off() { - assert!(requires_intrinsic_structural_tag( - Some("kimi_k2"), - &ToolChoice::Required - )); - assert!(requires_intrinsic_structural_tag( - Some("kimi_k2"), - &ToolChoice::Named("get_weather".to_string()) - )); - } - - // Durable registry-parity property: every parser/tool_choice combination the - // intrinsic predicate recognizes must have a real, registered - // `StructuralTagBuilder` in the parser registry. If a future edit registers a - // new intrinsic-family alias, or de-registers an existing one, without keeping - // both in lockstep, `structural_tag_decision` would otherwise report `Required` - // with no way to actually build the tag — this is the exact gap the shared - // decision owner exists to close. No global-state mutation, no scratch worktree, - // runs against the real production registry. - #[test] - fn every_intrinsic_parser_choice_has_a_registered_structural_tag_builder() { - let cases: &[(&str, ToolChoice)] = &[ - ("kimi_k2", ToolChoice::Required), - ("kimi_k2", ToolChoice::Named("get_weather".to_string())), - ("kimi_k3", ToolChoice::Named("get_weather".to_string())), - ("kimi-k3", ToolChoice::Named("get_weather".to_string())), - ]; - for (parser, choice) in cases { + fn kimi_k3_reasoning_gate_avoids_duplicate_prefix_per_request() { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: Some(serde_json::json!({ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + })), + strict: None, + }]; + for (gate, require_reasoning, prompt_injected_reasoning, choice, expects_prefix) in [ + (Some(true), true, true, ToolChoice::Required, false), + (None, true, true, ToolChoice::Required, true), + (Some(false), true, true, ToolChoice::Required, true), + (Some(true), false, true, ToolChoice::Auto, true), + (Some(true), true, false, ToolChoice::Required, false), + ( + Some(true), + false, + false, + ToolChoice::Named("get_weather".to_string()), + false, + ), + ] { + let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); + mdc.runtime_config.structural_tag_mode = StructuralTagMode::On; + mdc.runtime_config.structural_tag_scope = StructuralTagScope::Always; + mdc.runtime_config.tool_call_parser = Some("kimi_k3".to_string()); + if let Some(gate) = gate { + mdc.runtime_config + .set_engine_specific(TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, gate) + .unwrap(); + } + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let mut request = preprocessed_request(); + request.require_reasoning = require_reasoning; assert!( - requires_intrinsic_structural_tag(Some(parser), choice), - "test fixture drifted: '{parser}' + {choice:?} is no longer intrinsic" + preprocessor + .apply_tool_choice_structural_tag( + &choice, + &tools, + None, + prompt_injected_reasoning, + &mut request, + ) + .unwrap(), + "gate={gate:?}, require_reasoning={require_reasoning}, choice={choice:?}" ); - assert!( - OpenAIPreprocessor::structural_tag_builder_for_parser(parser).is_some(), - "'{parser}' is intrinsic per the predicate but the parser registry has \ - no structural-tag builder for it — the predicate and the registry have \ - drifted apart" + let tag = request + .sampling_options + .guided_decoding + .unwrap() + .structural_tag + .unwrap(); + let has_prefix = tag["format"]["elements"][0]["end"] == "<|close|>think<|sep|>"; + assert_eq!( + has_prefix, expects_prefix, + "gate={gate:?}, require_reasoning={require_reasoning}, prompt_injected_reasoning={prompt_injected_reasoning}, choice={choice:?}" ); + assert!(tag.to_string().contains("<|open|>tools<|sep|>")); } } #[test] - fn other_choices_and_parsers_still_follow_the_global_mode() { - assert!(!requires_intrinsic_structural_tag( + fn kimi_forced_choices_are_classified_as_native_format_fallbacks() { + let named = ToolChoice::Named("get_weather".to_string()); + assert!(requires_native_tool_call_format( + Some("kimi_k2"), + &ToolChoice::Required + )); + assert!(requires_native_tool_call_format(Some("kimi_k2"), &named)); + assert!(requires_native_tool_call_format( Some("kimi_k3"), &ToolChoice::Required )); - assert!(!requires_intrinsic_structural_tag( + assert!(requires_native_tool_call_format(Some("kimi_k3"), &named)); + assert!(requires_native_tool_call_format(Some("kimi-k3"), &named)); + assert!(!requires_native_tool_call_format( + Some("kimi_k3"), + &ToolChoice::Auto + )); + assert!(!requires_native_tool_call_format( Some("hermes"), &ToolChoice::Named("get_weather".to_string()) )); - assert!(!requires_intrinsic_structural_tag( + assert!(!requires_native_tool_call_format( Some("kimi_k2"), &ToolChoice::Auto )); } #[test] - fn kimi_k2_required_installs_native_tag_when_global_mode_is_off() { + fn kimi_forced_choices_keep_native_tags_when_global_mode_is_off() { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); - let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; - mdc.runtime_config.tool_call_parser = Some("kimi_k2".to_string()); - let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); let tools = [ToolDefinition { name: "get_weather".to_string(), parameters: Some(serde_json::json!({ @@ -506,39 +552,48 @@ mod tests { })), strict: None, }]; - let mut request = preprocessed_request(); - - let applied = preprocessor - .apply_tool_choice_structural_tag( - &ToolChoice::Required, - &tools, - None, - false, - &mut request, - ) - .unwrap(); + for (parser, choice, marker) in [ + ( + "kimi_k2", + ToolChoice::Required, + "<|tool_calls_section_begin|>", + ), + ( + "kimi_k2", + ToolChoice::Named("get_weather".to_string()), + "<|tool_calls_section_begin|>", + ), + ( + "kimi_k3", + ToolChoice::Named("get_weather".to_string()), + "<|open|>call", + ), + ( + "kimi-k3", + ToolChoice::Named("get_weather".to_string()), + "<|open|>call", + ), + ] { + let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); + mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; + mdc.runtime_config.tool_call_parser = Some(parser.to_string()); + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let mut request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_structural_tag(&choice, &tools, None, false, &mut request) + .unwrap(); - assert!(applied); - let format = &request - .sampling_options - .guided_decoding - .as_ref() - .unwrap() - .structural_tag - .as_ref() - .unwrap()["format"]; - assert_eq!(format["type"], "sequence"); - assert_eq!( - format["elements"][0]["value"], - "<|tool_calls_section_begin|>" - ); - assert_eq!(format["elements"][1]["type"], "tags_with_separator"); - assert_eq!(format["elements"][1]["at_least_one"], true); - assert_eq!( - format["elements"][1]["tags"][0]["begin"], - "<|tool_call_begin|>functions.get_weather:" - ); - assert_eq!(format["elements"][2]["value"], "<|tool_calls_section_end|>"); + assert!(applied, "{parser} + {choice:?} must retain its native tag"); + let tag = request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.structural_tag.as_ref()) + .expect("forced Kimi choice must install a native structural tag"); + assert!(tag.to_string().contains(marker)); + assert!(tag.to_string().contains("get_weather")); + } } #[test] @@ -686,6 +741,58 @@ mod tests { assert!(!applied); } + #[test] + fn global_mode_off_is_authoritative_for_kimi_k3_auto() { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); + mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; + mdc.runtime_config.structural_tag_scope = StructuralTagScope::Auto; + mdc.runtime_config.tool_call_parser = Some("kimi_k3".to_string()); + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: Some(serde_json::json!({ + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"] + })), + // K3 auto must be constrained even when the caller does not opt in + // to OpenAI strict schema enforcement. + strict: None, + }]; + let mut request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_structural_tag(&ToolChoice::Auto, &tools, None, false, &mut request) + .unwrap(); + + assert!(!applied); + assert!(request.sampling_options.guided_decoding.is_none()); + } + + #[test] + fn always_scope_activates_auto_when_strict_is_omitted() { + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: None, + strict: None, + }]; + + assert!(OpenAIPreprocessor::should_apply_tool_call_format( + StructuralTagScope::Always, + &ToolChoice::Auto, + &tools, + None, + )); + assert!(!OpenAIPreprocessor::should_apply_tool_call_format( + StructuralTagScope::Auto, + &ToolChoice::Auto, + &tools, + None, + )); + } + #[test] fn tool_choice_none_skips_ban_only_when_prompt_excludes_tools() { let tools = [ToolDefinition { diff --git a/lib/llm/src/preprocessor/tool_choice.rs b/lib/llm/src/preprocessor/tool_choice.rs index e14ca420239f..403d488dac73 100644 --- a/lib/llm/src/preprocessor/tool_choice.rs +++ b/lib/llm/src/preprocessor/tool_choice.rs @@ -5,6 +5,7 @@ use std::borrow::Cow; +use crate::preprocessor::structural_tag::requires_native_tool_call_format; use crate::preprocessor::{OpenAIPreprocessor, PreprocessedRequest}; use crate::protocols::openai::chat_completions::NvCreateChatCompletionRequest; use crate::protocols::openai::tools::{ @@ -58,11 +59,10 @@ impl OpenAIPreprocessor { /// Apply guided decoding for OpenAI tool-choice requests. /// /// Structural tags are preferred when enabled and supported by the configured - /// tool-call parser. Supported K2 forced requests and named K3 requests - /// intrinsically use their native structural tags because generic JSON cannot - /// represent their tool calls. Other forced choices fall back to the legacy - /// JSON-schema constraint when structural tags are not applied, except K3 - /// required requests, which stay on the prompt-level XTML path. + /// tool-call parser. When disabled or unavailable, ordinary forced choices + /// retain the legacy JSON-schema fallback. Kimi's marker-delimited forced + /// formats cannot be represented by that fallback and remain on their + /// prompt/parser best-effort path instead. pub(super) fn apply_tool_choice_guided_decoding( &self, request: &NvCreateChatCompletionRequest, @@ -81,6 +81,7 @@ impl OpenAIPreprocessor { ); let has_explicit_guided_decoding = has_explicit_guided_decoding(request); let has_response_format_constraint = has_response_format_constraint(request); + let parser_tool_choice = convert_tool_choice(tool_choice); if is_forced_tool_choice && has_explicit_guided_decoding { return Err(invalid_argument(concat!( @@ -106,7 +107,7 @@ impl OpenAIPreprocessor { } if self.apply_tool_choice_structural_tag( - &convert_tool_choice(tool_choice), + &parser_tool_choice, &convert_tools(tools.as_ref()), request.inner.parallel_tool_calls, prompt_injected_reasoning, @@ -115,22 +116,17 @@ impl OpenAIPreprocessor { return Ok(GuidedToolConstraint::StructuralTag); } - let uses_kimi_k3_parser = uses_kimi_k3_parser( - self.tool_call_parser.as_deref(), - self.runtime_config.reasoning_parser.as_deref(), - ); - if is_forced_tool_choice && uses_kimi_k3_parser { - if matches!(tool_choice, ChatCompletionToolChoiceOption::Named(_)) { - return Err(invalid_argument( - "named tool choice for Kimi K3 requires --dyn-tool-call-parser kimi_k3 \ - with XTML structural-tag support", - )); - } - - // K3's prompt-level required instruction produces an XTML `tools` - // channel. Generic JSON guided decoding would constrain the wrong - // wire format and prevent the Rust K3 parser from seeing it. No JSON - // schema is installed, so this is NOT a guided-JSON request. + let requires_native_format = + requires_native_tool_call_format(self.tool_call_parser.as_deref(), &parser_tool_choice) + || requires_native_tool_call_format( + self.runtime_config.reasoning_parser.as_deref(), + &parser_tool_choice, + ); + if is_forced_tool_choice && requires_native_format { + tracing::debug!( + tool_choice = ?parser_tool_choice, + "Structural tag was not applied and the parser uses a native tool-call format; using prompt/parser compatibility fallback" + ); return Ok(GuidedToolConstraint::None); } @@ -287,8 +283,13 @@ pub(crate) fn guided_tool_constraint_with_effective_tools( if has_explicit_guided_decoding(request) { return Ok(GuidedToolConstraint::None); } - // K3 forced requests are served by a prompt-level XTML instruction; no JSON schema. - if uses_kimi_k3_parser(tool_call_parser, reasoning_parser) { + let parser_tool_choice = convert_tool_choice(tool_choice); + // Kimi forced requests use marker-delimited native formats that generic JSON + // guidance cannot represent. Keep this compatibility path aligned with + // request preprocessing for either configured parser slot. + if requires_native_tool_call_format(tool_call_parser, &parser_tool_choice) + || requires_native_tool_call_format(reasoning_parser, &parser_tool_choice) + { return Ok(GuidedToolConstraint::None); } // Validate the forced choice against the actual tools the same way @@ -306,15 +307,6 @@ pub(crate) fn guided_tool_constraint_with_effective_tools( } } -/// True when either configured parser is Kimi K3. -/// -/// K3 forced requests are served by a prompt-level XTML instruction, so they must -/// NOT be reported as guided JSON even though their `tool_choice` is forced. -fn uses_kimi_k3_parser(tool_call_parser: Option<&str>, reasoning_parser: Option<&str>) -> bool { - let is_k3 = |parser: &str| matches!(parser, "kimi_k3" | "kimi-k3"); - tool_call_parser.is_some_and(is_k3) || reasoning_parser.is_some_and(is_k3) -} - /// Map a forced `tool_choice` onto the guided-JSON parser constraint for its grammar. /// /// Only reachable once `get_tool_choice_guidance_from_tools` produced either JSON or @@ -334,7 +326,12 @@ fn installed_json_constraint(tool_choice: &ChatCompletionToolChoiceOption) -> Gu #[cfg(test)] mod tests { + use std::{path::PathBuf, sync::Arc}; + use super::*; + use crate::local_model::runtime_config::StructuralTagMode; + use crate::model_card::ModelDeploymentCard; + use crate::protocols::common::{OutputOptions, SamplingOptions, StopConditions}; use dynamo_protocols::types::{ChatCompletionNamedToolChoice, FunctionName}; use serde_json::{Value, json}; @@ -364,6 +361,26 @@ mod tests { }]) } + fn preprocessor(parser: &str, mode: StructuralTagMode) -> Arc { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); + mdc.runtime_config.structural_tag_mode = mode; + mdc.runtime_config.tool_call_parser = Some(parser.to_string()); + OpenAIPreprocessor::new(mdc).unwrap() + } + + fn preprocessed_request() -> PreprocessedRequest { + PreprocessedRequest::builder() + .model("test-model".to_string()) + .token_ids(Vec::new()) + .stop_conditions(StopConditions::default()) + .sampling_options(SamplingOptions::default()) + .output_options(OutputOptions::default()) + .build() + .unwrap() + } + #[test] fn tool_call_parsing_requires_tools() { assert!(!OpenAIPreprocessor::tool_call_parsing_enabled(&request( @@ -650,18 +667,35 @@ mod tests { } #[test] - fn kimi_k3_is_detected_from_either_parser_slot() { - assert!(uses_kimi_k3_parser(Some("kimi_k3"), None)); - assert!(uses_kimi_k3_parser(Some("kimi-k3"), None)); - assert!(uses_kimi_k3_parser(None, Some("kimi_k3"))); - assert!(uses_kimi_k3_parser(None, Some("kimi-k3"))); - } + fn native_format_forced_choices_install_no_compatibility_json_constraint() { + let requests = [ + request(json!({ + "tools": tools(), + "tool_choice": "required" + })), + request(json!({ + "tools": tools(), + "tool_choice": { + "type": "function", + "function": {"name": "get_weather"} + } + })), + ]; - #[test] - fn non_k3_parsers_are_not_mistaken_for_k3() { - assert!(!uses_kimi_k3_parser(None, None)); - assert!(!uses_kimi_k3_parser(Some("kimi_k2"), Some("qwen3"))); - assert!(!uses_kimi_k3_parser(Some("qwen3_coder"), None)); + for request in &requests { + for (tool_call_parser, reasoning_parser) in [ + (Some("kimi_k2"), None), + (None, Some("kimi_k2")), + (Some("kimi_k3"), None), + (None, Some("kimi-k3")), + ] { + assert_eq!( + guided_tool_constraint(request, tool_call_parser, reasoning_parser, false,) + .expect("native-format choice is valid"), + GuidedToolConstraint::None, + ); + } + } } #[test] @@ -679,6 +713,136 @@ mod tests { } } + #[test] + fn structural_tag_off_preserves_kimi_k2_required_native_tag() { + let request = request(json!({ + "tools": tools(), + "tool_choice": "required" + })); + let preprocessor = preprocessor("kimi_k2", StructuralTagMode::Off); + let mut common_request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_guided_decoding(&request, &mut common_request, false) + .unwrap(); + + assert_eq!(applied, GuidedToolConstraint::StructuralTag); + assert!( + common_request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.structural_tag.as_ref()) + .is_some() + ); + } + + #[test] + fn structural_tag_off_preserves_kimi_k3_named_native_tag() { + let request = request(json!({ + "tools": tools(), + "tool_choice": { + "type": "function", + "function": {"name": "get_weather"} + } + })); + let preprocessor = preprocessor("kimi_k3", StructuralTagMode::Off); + let mut common_request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_guided_decoding(&request, &mut common_request, false) + .unwrap(); + + assert_eq!(applied, GuidedToolConstraint::StructuralTag); + assert!( + common_request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.structural_tag.as_ref()) + .is_some() + ); + } + + #[test] + fn structural_tag_off_preserves_generic_forced_tool_fallback() { + let request = request(json!({ + "tools": tools(), + "tool_choice": "required" + })); + let preprocessor = preprocessor("hermes", StructuralTagMode::Off); + let mut common_request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_guided_decoding(&request, &mut common_request, false) + .unwrap(); + + assert_eq!(applied, GuidedToolConstraint::GuidedJsonRequired); + assert!( + common_request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.json.as_ref()) + .is_some() + ); + } + + #[test] + fn kimi_k3_builder_error_uses_native_fallback() { + let request = request(json!({ + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "strict": true, + "parameters": { + "type": "object", + "properties": { + "value": { + "type": "string", + "$id": "urn:example:scoped", + "$defs": { + "allowed": {"type": "string", "enum": ["safe"]} + }, + "allOf": [{"$ref": "#/$defs/allowed"}] + } + }, + "required": ["value"] + } + } + }], + "tool_choice": "auto" + })); + let preprocessor = preprocessor("kimi_k3", StructuralTagMode::On); + let tools = convert_tools(request.inner.tools.as_deref().unwrap()); + let ctx = dynamo_parsers::tool_calling::ToolCallFormatBuildContext { + tool_choice: &ToolChoice::Auto, + tools: &tools, + parallel_tool_calls: request.inner.parallel_tool_calls, + schema_mode: preprocessor.runtime_config.structural_tag_schema, + starts_in_reasoning: false, + }; + // A scoped string reference cannot be safely rendered as raw XTML. + // Verify an actual builder error, rather than a policy skip or Ok(None). + let error = dynamo_parsers::tool_calling::StructuralTagBuilder::KimiK3 + .build_tool_call_format(&ctx) + .expect_err("scoped string reference must reach the builder-error fallback"); + let message = format!("{error:#}"); + assert!( + message.contains("value") && message.contains("string"), + "{message}" + ); + + let mut common_request = preprocessed_request(); + let applied = preprocessor + .apply_tool_choice_guided_decoding(&request, &mut common_request, false) + .expect("builder errors must allow the request to continue without a structural tag"); + + assert_eq!(applied, GuidedToolConstraint::None); + assert!(common_request.sampling_options.guided_decoding.is_none()); + } + #[test] fn kimi_k3_forced_choices_accept_dynamic_system_tools() { for tool_choice in [ diff --git a/recipes/inkling/README.md b/recipes/inkling/README.md index d669378fa23d..0fc2414f680f 100644 --- a/recipes/inkling/README.md +++ b/recipes/inkling/README.md @@ -342,18 +342,18 @@ curl -s http://localhost:8000/v1/chat/completions \ }' ``` -**Keep both structured-output flags on the worker.** `tool_choice: "required"` and named -`tool_choice` are enforced through a structural-tag grammar that needs both flags the manifests -already set: +**Keep structured output support enabled on the worker.** `tool_choice: "required"` and named +`tool_choice` are enforced through a structural-tag grammar. The manifests set: ``` --dyn-enable-structural-tag --structured-outputs-config '{"enable_in_reasoning": true}' ``` -Drop the second and the grammar is silently inert on a reasoning model — HTTP 200, no error, and -the requested tool is not enforced. Both belong on the worker; the frontend does not accept -`--dyn-enable-structural-tag`. +`--dyn-enable-structural-tag` pins Dynamo's default-on behavior. The +`--structured-outputs-config` setting remains required for this reasoning model; remove it and the +grammar is silently inert — HTTP 200, no error, and the requested tool is not enforced. Both are +worker arguments; the frontend does not accept `--dyn-enable-structural-tag`. **Image and audio input are not supported.** The Dynamo vLLM runtime has no multimodal support for this model yet, so the vLLM GB300 profiles are text-only; use the SGLang B200 profile for image and diff --git a/tests/report_pytest_markers.py b/tests/report_pytest_markers.py index 5f643ea994f8..ef4f5da25c6c 100755 --- a/tests/report_pytest_markers.py +++ b/tests/report_pytest_markers.py @@ -165,6 +165,9 @@ "fsspec.implementations.dirfs", "sglang", "sglang.srt", + "sglang.srt.constrained", + "sglang.srt.constrained.reasoner_grammar_backend", + "sglang.srt.constrained.xgrammar_backend", "sglang.srt.entrypoints", "sglang.srt.entrypoints.openai", "sglang.srt.entrypoints.openai.protocol", @@ -191,6 +194,7 @@ "sglang.srt.disaggregation.utils", "sglang.srt.server_args", "sglang.srt.server_args_config_parser", + "xgrammar", "vllm", "vllm.config", "vllm.distributed",