diff --git a/Cargo.lock b/Cargo.lock index 7fd5eb396a49..e805e8574359 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2900,9 +2900,9 @@ dependencies = [ [[package]] name = "dynamo-parsers-v2" -version = "0.7.17" +version = "0.7.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f80b88ab86928219f3444822dc33c1ca170d0cc67beeb40d18cf7d6f287bad5e" +checksum = "f43e47b6a68353cc04d4dfe19c69414a5884c9963b72c68937d019998c5ef72e" dependencies = [ "anyhow", "openai-harmony", diff --git a/Cargo.toml b/Cargo.toml index ef6089e1d7a6..f00c66e1a2d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -81,7 +81,7 @@ dynamo-parsers = { version = "=9.2.7" } # web: https://crates.io/crates/dynamo-parsers-v2/versions # sparse index: curl https://index.crates.io/dy/na/dynamo-parsers-v2 (one JSON line per release; read the "vers" field) # Native count-before-string early streaming still needs a frontend-crates fix. -dynamo-parsers-v2 = { version = "=0.7.17" } +dynamo-parsers-v2 = { version = "=0.7.21" } dynamo-backend-common = { path = "lib/backend-common", version = "1.6.0" } dynamo-sidecar-common = { path = "lib/sidecar/common", version = "1.6.0" } diff --git a/components/src/dynamo/common/backend/tests/test_backend_bindings.py b/components/src/dynamo/common/backend/tests/test_backend_bindings.py index 714922192b92..480ee469a7a5 100644 --- a/components/src/dynamo/common/backend/tests/test_backend_bindings.py +++ b/components/src/dynamo/common/backend/tests/test_backend_bindings.py @@ -160,6 +160,21 @@ def test_worker_config_accepts_parser_runtime_settings(): ) +@pytest.mark.none +def test_structural_tag_bindings_reject_unknown_config_fields(): + from dynamo.llm import ModelRuntimeConfig + + backend.WorkerConfig(namespace="dynamo", structural_tag={"scope": "always"}) + runtime_config = ModelRuntimeConfig() + runtime_config.set_structural_tag({"scope": "always"}) + + with pytest.raises(ValueError, match="unknown field `scpoe`"): + backend.WorkerConfig(namespace="dynamo", structural_tag={"scpoe": "always"}) + + with pytest.raises(ValueError, match="unknown field `scpoe`"): + runtime_config.set_structural_tag({"scpoe": "always"}) + + def test_worker_config_preserves_legacy_positional_argument_order(): """New optional fields must be appended after every existing argument.""" backend.WorkerConfig( @@ -247,9 +262,14 @@ def test_python_worker_config_from_runtime_config_copies_parser_settings(): runtime_cfg.dyn_default_thinking_mode = "disabled" runtime_cfg.exclude_tools_when_tool_choice_none = False runtime_cfg.enable_local_indexer = False - runtime_cfg.dyn_enable_structural_tag = True - runtime_cfg.dyn_structural_tag_scope = "always" - runtime_cfg.dyn_structural_tag_schema = "strict" + runtime_cfg.structural_tag = { + "scope": "always", + "schema": "strict", + "allow_tool_calls_with_structured_output": True, + "exclude_special_tokens": False, + "reasoning_boundary": "structural_tag", + "tool_arguments_any_order": False, + } # MagicMock auto-attrs would be rejected as a foreign type by the # strict coercer; pin them to None. runtime_cfg.disaggregation_mode = None @@ -262,9 +282,7 @@ def test_python_worker_config_from_runtime_config_copies_parser_settings(): assert config.default_thinking_mode == "disabled" assert config.exclude_tools_when_tool_choice_none is False assert config.enable_local_indexer is False - assert config.structural_tag_mode == "on" - assert config.structural_tag_scope == "always" - assert config.structural_tag_schema == "strict" + assert config.structural_tag == runtime_cfg.structural_tag @pytest.mark.unified @@ -285,9 +303,7 @@ class _BareRuntime: assert cfg.use_kv_events is False assert cfg.custom_jinja_template is None assert cfg.default_thinking_mode is None - assert cfg.structural_tag_mode == "off" - assert cfg.structural_tag_scope == "auto" - assert cfg.structural_tag_schema == "auto" + assert cfg.structural_tag is None @pytest.mark.unified diff --git a/components/src/dynamo/common/backend/worker.py b/components/src/dynamo/common/backend/worker.py index 2a8d0550a11e..c507c1b16e89 100644 --- a/components/src/dynamo/common/backend/worker.py +++ b/components/src/dynamo/common/backend/worker.py @@ -131,9 +131,7 @@ class WorkerConfig: # Operator override; when set, the Rust Worker uses this instead of # `engine.health_check_payload()`. Populated by `from_runtime_config`. health_check_payload: Optional[dict] = None - structural_tag_mode: str = "off" - structural_tag_scope: str = "auto" - structural_tag_schema: str = "auto" + structural_tag: Optional[dict[str, object]] = None # When True, this worker declares an upstream Encode peer in its # topology `needs`. Meaningful only on AGGREGATED/PREFILL roles; # the Rust validator rejects DECODE/ENCODE + True with InvalidArgument. @@ -185,17 +183,7 @@ def from_runtime_config( ), "enable_local_indexer": getattr(runtime_cfg, "enable_local_indexer", True), "enable_kv_routing": getattr(runtime_cfg, "enable_kv_routing", True), - "structural_tag_mode": ( - "on" - if getattr(runtime_cfg, "dyn_enable_structural_tag", False) - else "off" - ), - "structural_tag_scope": getattr( - runtime_cfg, "dyn_structural_tag_scope", "auto" - ), - "structural_tag_schema": getattr( - runtime_cfg, "dyn_structural_tag_schema", "auto" - ), + "structural_tag": getattr(runtime_cfg, "structural_tag", None), "route_to_encoder": getattr(runtime_cfg, "route_to_encoder", False), } # Skip the probe when an override is supplied so callers with a foreign @@ -277,9 +265,7 @@ async def run(self) -> None: self.config.disaggregation_mode ), health_check_payload=self.config.health_check_payload, - structural_tag_mode=self.config.structural_tag_mode, - structural_tag_scope=self.config.structural_tag_scope, - structural_tag_schema=self.config.structural_tag_schema, + structural_tag=self.config.structural_tag, runtime=runtime_cfg, route_to_encoder=self.config.route_to_encoder, media_decoder=self.config.media_decoder, diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/components/src/dynamo/common/configuration/groups/runtime_args.py index 30970ef35986..0c9e897cc549 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/components/src/dynamo/common/configuration/groups/runtime_args.py @@ -6,12 +6,20 @@ import argparse import logging import os -from typing import List, Optional +from importlib.metadata import version as distribution_version +from typing import List, Literal, Optional + +import msgspec +from packaging.version import Version from dynamo._core import get_reasoning_parser_names, get_tool_parser_names from dynamo.common.configuration.arg_group import ArgGroup from dynamo.common.configuration.config_base import ConfigBase -from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument +from dynamo.common.configuration.utils import ( + add_argument, + add_negatable_bool_argument, + env_or_default, +) from dynamo.common.utils.namespace import get_worker_namespace from dynamo.common.utils.output_modalities import OutputModality @@ -20,6 +28,30 @@ _fpm_trace_invalid_warning_emitted = False +class StructuralTagConfig(msgspec.Struct, forbid_unknown_fields=True): + scope: Literal["auto", "always"] = "always" + schema: Literal["auto", "strict"] = "auto" + allow_tool_calls_with_structured_output: bool = False + exclude_special_tokens: Optional[bool] = None + reasoning_boundary: Literal["auto", "structural_tag", "backend"] = "auto" + tool_arguments_any_order: bool = False + + +_STRUCTURAL_TAG_CONFIG_DECODER = msgspec.json.Decoder(type=StructuralTagConfig) + + +def _parse_structural_tag(value: str) -> StructuralTagConfig | bool: + normalized = value.strip().lower() + if normalized in {"true", "1", "yes", "on"}: + return StructuralTagConfig() + if normalized in {"false", "0", "no", "off"}: + return False + try: + return _STRUCTURAL_TAG_CONFIG_DECODER.decode(value) + except msgspec.DecodeError as error: + raise argparse.ArgumentTypeError(str(error)) from error + + class DynamoRuntimeConfig(ConfigBase): """Configuration for Dynamo runtime (common across all backends).""" @@ -40,9 +72,11 @@ class DynamoRuntimeConfig(ConfigBase): dyn_reasoning_parser: Optional[str] = None dyn_default_thinking_mode: Optional[str] = None exclude_tools_when_tool_choice_none: bool = True - dyn_enable_structural_tag: bool = True - dyn_structural_tag_scope: str = "always" - dyn_structural_tag_schema: str = "auto" + dyn_enable_structural_tag: Optional[bool] = None + dyn_structural_tag_scope: Optional[Literal["auto", "always"]] = None + dyn_structural_tag_schema: Optional[Literal["auto", "strict"]] = None + dyn_structural_tag: Optional[StructuralTagConfig | bool] = None + structural_tag: Optional[dict[str, object]] = None custom_jinja_template: Optional[str] = None endpoint_types: str dump_config_to: Optional[str] = None @@ -75,6 +109,7 @@ class DynamoRuntimeConfig(ConfigBase): def validate(self) -> None: self.namespace = get_worker_namespace(self.namespace) + self.structural_tag = resolve_structural_tag_config(self) # The Rust FPM sink reads this setting from the process environment. # Canonicalize the resolved CLI/env value before the runtime or backend @@ -289,40 +324,38 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: help="Exclude tool definitions from the chat template when tool_choice='none'. " "Prevents models from generating raw XML tool calls in the content field.", ) - add_negatable_bool_argument( - g, - flag_name="--dyn-enable-structural-tag", - env_var="DYN_ENABLE_STRUCTURAL_TAG", - default=True, - help="Enable structural tag guided decoding for tool calls when the configured " - "parser and backend support it. Disabling this flag suppresses optional " - "guidance; Rust retains native tags for Kimi K2 required/named and " - "Kimi K3 named tool choices. Python vLLM and SGLang respect the opt-out. " - "Unsupported schema constructs can cause backend grammar compilation errors.", + g.add_argument( + "--dyn-enable-structural-tag", + action=argparse.BooleanOptionalAction, + default=env_or_default("DYN_ENABLE_STRUCTURAL_TAG", None, value_type=bool), + help=argparse.SUPPRESS, ) - add_argument( - g, - flag_name="--dyn-structural-tag-scope", - env_var="DYN_STRUCTURAL_TAG_SCOPE", - default="always", + g.add_argument( + "--dyn-structural-tag-scope", + default=env_or_default("DYN_STRUCTURAL_TAG_SCOPE", None, value_type=str), choices=["auto", "always"], - help="Controls when structural tags are activated. " - "'auto': for required/named tool_choice, and if any tool has strict=true " - "or parallel_tool_calls is false. " - "'always': also for auto without those conditions. " - "tool_choice none is unaffected by auto vs always.", + help=argparse.SUPPRESS, + ) + g.add_argument( + "--dyn-structural-tag-schema", + default=env_or_default("DYN_STRUCTURAL_TAG_SCHEMA", None, value_type=str), + choices=["auto", "strict"], + help=argparse.SUPPRESS, ) add_argument( g, - flag_name="--dyn-structural-tag-schema", - env_var="DYN_STRUCTURAL_TAG_SCHEMA", - default="auto", - choices=["auto", "strict"], - help="Controls parameter schema strictness inside structural tags. " - "'auto': declared parameter schema for tools with strict omitted or true; " - "tools with strict=false keep the native tool envelope constrained but use " - "schema-unconstrained arguments. " - "'strict': declared parameter schema for all tools, overriding strict=false.", + flag_name="--dyn-structural-tag", + env_var="DYN_STRUCTURAL_TAG", + default=None, + arg_type=_parse_structural_tag, + nargs="?", + const="true", + help="Configure default-on structural tag guided decoding with a JSON object, " + "or pass false to disable optional guidance. Rust retains native tags for " + "Kimi K2 required/named and Kimi K3 named choices; Python vLLM and SGLang " + "respect the opt-out. Schema auto enforces strict omitted or true; schema " + "strict enforces every tool, overriding strict=false. Unsupported schema " + "constructs can cause backend grammar compilation errors.", ) add_argument( g, @@ -529,3 +562,79 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: default=None, help="Path to PEM private key for the NATS client certificate (mTLS).", ) + + +def _validate_xgrammar_any_order_support() -> None: + xgrammar_version = Version(distribution_version("xgrammar")) + if xgrammar_version < Version("0.2.3"): + raise ValueError( + "tool_arguments_any_order requires XGrammar >= 0.2.3; " + f"found {xgrammar_version}" + ) + + +def resolve_structural_tag_config( + runtime_config: DynamoRuntimeConfig, +) -> Optional[dict[str, object]]: + """Normalize the public setting and hidden legacy flags.""" + + structural_tag_setting = runtime_config.dyn_structural_tag + if structural_tag_setting is True: + structural_tag_setting = StructuralTagConfig() + legacy_enable = runtime_config.dyn_enable_structural_tag + legacy_scope = runtime_config.dyn_structural_tag_scope + legacy_schema = runtime_config.dyn_structural_tag_schema + + legacy_options = [] + if legacy_enable is not None: + legacy_options.append( + "--dyn-enable-structural-tag" + if legacy_enable + else "--no-dyn-enable-structural-tag" + ) + if legacy_scope is not None: + legacy_options.append("--dyn-structural-tag-scope") + if legacy_schema is not None: + legacy_options.append("--dyn-structural-tag-schema") + if legacy_options: + logger.warning( + "%s deprecated; use --dyn-structural-tag instead", + ", ".join(legacy_options), + ) + + if structural_tag_setting is not None: + enabled = structural_tag_setting is not False + conflicting_options = [] + if legacy_enable is not None and legacy_enable != enabled: + conflicting_options.append( + "--dyn-enable-structural-tag" + if legacy_enable + else "--no-dyn-enable-structural-tag" + ) + if legacy_scope is not None: + conflicting_options.append("--dyn-structural-tag-scope") + if legacy_schema is not None: + conflicting_options.append("--dyn-structural-tag-schema") + if conflicting_options: + raise ValueError( + "--dyn-structural-tag cannot be combined with legacy option(s): " + + ", ".join(conflicting_options) + ) + if not enabled: + return None + if ( + isinstance(structural_tag_setting, StructuralTagConfig) + and structural_tag_setting.tool_arguments_any_order + ): + _validate_xgrammar_any_order_support() + return msgspec.to_builtins(structural_tag_setting) + + if legacy_enable is False: + return None + + return msgspec.to_builtins( + StructuralTagConfig( + scope=legacy_scope if legacy_scope is not None else "always", + schema=legacy_schema if legacy_schema is not None else "auto", + ) + ) diff --git a/components/src/dynamo/common/tests/configuration/test_runtime_args.py b/components/src/dynamo/common/tests/configuration/test_runtime_args.py index d1a011815c35..53dcd0c76add 100644 --- a/components/src/dynamo/common/tests/configuration/test_runtime_args.py +++ b/components/src/dynamo/common/tests/configuration/test_runtime_args.py @@ -32,32 +32,33 @@ def _parse_runtime_args(argv: list[str]) -> tuple[DynamoRuntimeConfig, str]: def test_structural_tags_default_on_for_supported_parsers(monkeypatch): - monkeypatch.delenv("DYN_ENABLE_STRUCTURAL_TAG", raising=False) - monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCOPE", raising=False) - monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCHEMA", raising=False) + _clear_structural_tag_env(monkeypatch) config, _ = _parse_runtime_args([]) - assert config.dyn_enable_structural_tag is True - assert config.dyn_structural_tag_scope == "always" - assert config.dyn_structural_tag_schema == "auto" + assert config.structural_tag is not None + assert config.structural_tag["scope"] == "always" + assert config.structural_tag["schema"] == "auto" def test_structural_tag_schema_help_describes_omitted_strict(monkeypatch): - monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCHEMA", raising=False) + _clear_structural_tag_env(monkeypatch) _, help_text = _parse_runtime_args([]) + help_text = " ".join(help_text.split()) assert "strict omitted or true" in help_text assert "overriding strict=false" in help_text def test_structural_tag_global_opt_out(monkeypatch): + _clear_structural_tag_env(monkeypatch) monkeypatch.setenv("DYN_ENABLE_STRUCTURAL_TAG", "false") config, _ = _parse_runtime_args([]) assert config.dyn_enable_structural_tag is False + assert config.structural_tag is None def test_fpm_trace_defaults_disabled(monkeypatch): @@ -173,6 +174,182 @@ def test_fpm_trace_help_lists_flag_and_env(monkeypatch): assert "DYN_FPM_TRACE" in help_text +def _clear_structural_tag_env(monkeypatch): + for name in ( + "DYN_ENABLE_STRUCTURAL_TAG", + "DYN_STRUCTURAL_TAG_SCOPE", + "DYN_STRUCTURAL_TAG_SCHEMA", + "DYN_STRUCTURAL_TAG", + ): + monkeypatch.delenv(name, raising=False) + + +def test_default_structural_tag_policy_does_not_warn_about_legacy_options( + monkeypatch, caplog +): + _clear_structural_tag_env(monkeypatch) + + config, _ = _parse_runtime_args([]) + + assert config.structural_tag is not None + assert config.structural_tag["scope"] == "always" + assert "deprecated" not in caplog.text + assert config.dyn_enable_structural_tag is None + + +def test_structural_tag_flag_uses_default_config(monkeypatch): + _clear_structural_tag_env(monkeypatch) + + config, help_text = _parse_runtime_args(["--dyn-structural-tag"]) + + assert config.structural_tag == { + "scope": "always", + "schema": "auto", + "allow_tool_calls_with_structured_output": False, + "exclude_special_tokens": None, + "reasoning_boundary": "auto", + "tool_arguments_any_order": False, + } + assert "DYN_STRUCTURAL_TAG" in help_text + assert "--dyn-enable-structural-tag" not in help_text + + +@pytest.mark.parametrize( + "argv", [["--dyn-structural-tag", "false"], ["--no-dyn-enable-structural-tag"]] +) +def test_structural_tag_cli_opt_out(monkeypatch, argv): + _clear_structural_tag_env(monkeypatch) + config, _ = _parse_runtime_args(argv) + assert config.structural_tag is None + + +@pytest.mark.parametrize( + "argv, scope, schema", + [ + (["--dyn-structural-tag-scope", "auto"], "auto", "auto"), + (["--dyn-structural-tag-schema", "strict"], "always", "strict"), + ], +) +def test_legacy_tuning_alone_keeps_default_guidance_enabled( + monkeypatch, argv, scope, schema +): + _clear_structural_tag_env(monkeypatch) + config, _ = _parse_runtime_args(argv) + assert config.structural_tag is not None + assert config.structural_tag["scope"] == scope + assert config.structural_tag["schema"] == schema + + +def test_structural_tag_json_config_enables_and_fills_defaults(monkeypatch): + _clear_structural_tag_env(monkeypatch) + monkeypatch.setattr(runtime_args, "distribution_version", lambda _: "0.2.3") + + config, _ = _parse_runtime_args( + [ + "--dyn-structural-tag", + '{"scope":"always","schema":"strict",' + '"allow_tool_calls_with_structured_output":true,' + '"exclude_special_tokens":false,' + '"reasoning_boundary":"backend",' + '"tool_arguments_any_order":true}', + ] + ) + + assert config.structural_tag == { + "scope": "always", + "schema": "strict", + "allow_tool_calls_with_structured_output": True, + "exclude_special_tokens": False, + "reasoning_boundary": "backend", + "tool_arguments_any_order": True, + } + + +@pytest.mark.parametrize( + ("xgrammar_version", "supported"), [("0.2.1", False), ("0.2.3", True)] +) +def test_structural_tag_any_order_requires_supported_xgrammar( + monkeypatch, xgrammar_version, supported +): + _clear_structural_tag_env(monkeypatch) + monkeypatch.setattr( + runtime_args, "distribution_version", lambda _: xgrammar_version + ) + argv = ["--dyn-structural-tag", '{"tool_arguments_any_order":true}'] + + if supported: + config, _ = _parse_runtime_args(argv) + assert config.structural_tag["tool_arguments_any_order"] is True + else: + with pytest.raises(ValueError, match="requires XGrammar >= 0.2.3"): + _parse_runtime_args(argv) + + +def test_structural_tag_environment_is_supported(monkeypatch): + _clear_structural_tag_env(monkeypatch) + monkeypatch.setenv( + "DYN_STRUCTURAL_TAG", + '{"scope":"always","schema":"strict"}', + ) + + config, _ = _parse_runtime_args([]) + + assert config.structural_tag is not None + assert config.structural_tag["scope"] == "always" + assert config.structural_tag["schema"] == "strict" + + +def test_legacy_structural_tag_options_resolve_to_canonical_config(monkeypatch, caplog): + _clear_structural_tag_env(monkeypatch) + + with caplog.at_level(logging.WARNING, logger=runtime_args.__name__): + config, _ = _parse_runtime_args( + [ + "--dyn-enable-structural-tag", + "--dyn-structural-tag-scope", + "always", + "--dyn-structural-tag-schema", + "strict", + ] + ) + + assert config.structural_tag is not None + assert config.structural_tag["scope"] == "always" + assert config.structural_tag["schema"] == "strict" + assert "deprecated" in caplog.text + + +def test_structural_tag_json_rejects_legacy_tuning(monkeypatch): + _clear_structural_tag_env(monkeypatch) + + with pytest.raises(ValueError, match="cannot be combined"): + _parse_runtime_args( + [ + "--dyn-structural-tag", + "{}", + "--dyn-structural-tag-scope", + "always", + ] + ) + + +@pytest.mark.parametrize( + "raw_config", + [ + "{", + "[]", + '{"unknown":true}', + '{"scope":"sometimes"}', + '{"allow_tool_calls_with_structured_output":"yes"}', + ], +) +def test_structural_tag_json_config_is_strict(monkeypatch, raw_config): + _clear_structural_tag_env(monkeypatch) + + with pytest.raises(SystemExit): + _parse_runtime_args(["--dyn-structural-tag", raw_config]) + + @pytest.mark.parametrize("mode", ["enabled", "disabled"]) def test_default_thinking_mode_cli(mode, monkeypatch): monkeypatch.delenv("DYN_DEFAULT_THINKING_MODE", raising=False) diff --git a/components/src/dynamo/frontend/structural_tag_policy.py b/components/src/dynamo/frontend/structural_tag_policy.py index b54b7499eaca..1ade8acc19bb 100644 --- a/components/src/dynamo/frontend/structural_tag_policy.py +++ b/components/src/dynamo/frontend/structural_tag_policy.py @@ -14,6 +14,13 @@ def runtime_structural_tag_options( """Read structural-tag policy from a model-card runtime config.""" if not isinstance(runtime_config, dict): return "off", "auto", "auto" + # New cards project legacy fields for old readers. Prefer the canonical + # setting here, including its explicit disabled value, when present. + if "structural_tag" in runtime_config: + config = runtime_config["structural_tag"] + if not isinstance(config, dict): + return "off", "auto", "auto" + return "on", config.get("scope", "auto"), config.get("schema", "auto") return ( runtime_config.get("structural_tag_mode", "off"), runtime_config.get("structural_tag_scope", "auto"), diff --git a/components/src/dynamo/frontend/tests/test_structural_tag_policy.py b/components/src/dynamo/frontend/tests/test_structural_tag_policy.py index ce93dd3073c6..99e9c2a541c4 100644 --- a/components/src/dynamo/frontend/tests/test_structural_tag_policy.py +++ b/components/src/dynamo/frontend/tests/test_structural_tag_policy.py @@ -71,3 +71,24 @@ def test_runtime_structural_tag_options_read_model_card_policy(): "structural_tag_schema": "strict", } ) == ("on", "always", "strict") + + +@pytest.mark.parametrize( + "config, expected", + [ + ( + {"structural_tag": {"scope": "always", "schema": "strict"}}, + ("on", "always", "strict"), + ), + ( + {"structural_tag": None, "structural_tag_mode": "on"}, + ("off", "auto", "auto"), + ), + ( + {"structural_tag": {"scope": "always"}, "structural_tag_mode": "off"}, + ("on", "always", "auto"), + ), + ], +) +def test_canonical_structural_tag_policy_takes_precedence(config, expected): + assert runtime_structural_tag_options(config) == expected diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index 64514d486d1f..cf0063e7135a 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -1588,11 +1588,33 @@ async def chat_engine_factory( else: reasoning_parser_class = None default_thinking_mode = runtime_default_thinking_mode(mdc.runtime_config()) + runtime_config = mdc.runtime_config() ( structural_tag_mode, structural_tag_scope, structural_tag_schema, - ) = runtime_structural_tag_options(mdc.runtime_config()) + ) = runtime_structural_tag_options(runtime_config) + structural_tag = ( + runtime_config.get("structural_tag") + if isinstance(runtime_config, dict) + else None + ) + if isinstance(structural_tag, dict): + unsupported_options = [ + name + for name, default in ( + ("allow_tool_calls_with_structured_output", False), + ("exclude_special_tokens", None), + ("reasoning_boundary", "auto"), + ("tool_arguments_any_order", False), + ) + if structural_tag.get(name, default) != default + ] + if unsupported_options: + logger.warning( + "vLLM chat processor ignores unsupported structural-tag option(s): %s", + ", ".join(unsupported_options), + ) block_size = self.config.kv_cache_block_size or 16 diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 8b52e6ec88b4..8266a6f96312 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -425,6 +425,16 @@ def require_reasoning_kwargs( return kwargs +def supports_require_reasoning(engine: Any) -> bool: + """Return whether this SGLang engine supports the reasoning-gate argument.""" + if not hasattr(engine, "async_generate"): + return False + return "require_reasoning" in filter_supported_async_generate_kwargs( + engine, + {"require_reasoning": False}, + ) + + __all__ = [ "ConfigArgumentMerger", "add_sglang_cli_compat", @@ -439,5 +449,6 @@ def require_reasoning_kwargs( "require_reasoning_kwargs", "resolved_server_args", "sglang_uses_mla_backend", + "supports_require_reasoning", "supports_external_mm_hashes", ] diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index 79e15b7aafd6..c0ba8475b952 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -500,11 +500,7 @@ async def get_runtime_config( runtime_config.exclude_tools_when_tool_choice_none = ( dynamo_args.exclude_tools_when_tool_choice_none ) - runtime_config.set_structural_tag_mode( - "on" if dynamo_args.dyn_enable_structural_tag else "off" - ) - runtime_config.set_structural_tag_scope(dynamo_args.dyn_structural_tag_scope) - runtime_config.set_structural_tag_schema(dynamo_args.dyn_structural_tag_schema) + runtime_config.set_structural_tag(dynamo_args.structural_tag) # Decode workers don't create the WorkerKvQuery endpoint, so don't advertise local indexer is_decode_worker = server_args.disaggregation_mode == "decode" runtime_config.enable_local_indexer = ( diff --git a/components/src/dynamo/trtllm/workers/llm_worker.py b/components/src/dynamo/trtllm/workers/llm_worker.py index f247cd5dcd1b..0e9aa90515da 100644 --- a/components/src/dynamo/trtllm/workers/llm_worker.py +++ b/components/src/dynamo/trtllm/workers/llm_worker.py @@ -790,11 +790,7 @@ async def init_llm_worker( runtime_config.exclude_tools_when_tool_choice_none = ( config.exclude_tools_when_tool_choice_none ) - runtime_config.set_structural_tag_mode( - "on" if config.dyn_enable_structural_tag else "off" - ) - runtime_config.set_structural_tag_scope(config.dyn_structural_tag_scope) - runtime_config.set_structural_tag_schema(config.dyn_structural_tag_schema) + runtime_config.set_structural_tag(config.structural_tag) # Decode workers don't create the WorkerKvQuery endpoint, so don't advertise local indexer runtime_config.enable_local_indexer = ( config.enable_local_indexer diff --git a/components/src/dynamo/vllm/main.py b/components/src/dynamo/vllm/main.py index 7d806cdd4bf5..48f5db1e056e 100644 --- a/components/src/dynamo/vllm/main.py +++ b/components/src/dynamo/vllm/main.py @@ -918,11 +918,7 @@ async def register_vllm_model( runtime_config.exclude_tools_when_tool_choice_none = ( config.exclude_tools_when_tool_choice_none ) - runtime_config.set_structural_tag_mode( - "on" if config.dyn_enable_structural_tag else "off" - ) - runtime_config.set_structural_tag_scope(config.dyn_structural_tag_scope) - runtime_config.set_structural_tag_schema(config.dyn_structural_tag_schema) + runtime_config.set_structural_tag(config.structural_tag) # Propagate stream_interval so the frontend can respect --stream-interval. # set_engine_specific requires a JSON-encoded string (the Rust binding diff --git a/docs/fern/pages/reference/components/runtime-configuration.mdx b/docs/fern/pages/reference/components/runtime-configuration.mdx index fb7004ee22a0..3dc29a5e4560 100644 --- a/docs/fern/pages/reference/components/runtime-configuration.mdx +++ b/docs/fern/pages/reference/components/runtime-configuration.mdx @@ -13,7 +13,7 @@ subtitle: Field reference for the cross-cutting Dynamo runtime CLI flags and env ## How the config is loaded -Unless a field is marked environment-only, it has both a CLI flag and an environment variable. The CLI flag takes precedence; the environment variable is the fallback. Boolean fields are negatable — `--dyn-enable-structural-tag` sets it on, `--no-dyn-enable-structural-tag` sets it off. +Unless a field is marked environment-only, it has both a CLI flag and an environment variable. The CLI flag takes precedence; the environment variable is the fallback. Boolean fields are negatable — for example, `--enable-local-indexer` sets it on and `--no-enable-local-indexer` sets it off. @@ -296,26 +296,10 @@ address. Environment variable: `DYN_EXCLUDE_TOOLS_WHEN_TOOL_CHOICE_NONE` - - Enable structural tag guided decoding for tool calls when the configured parser and backend support it. The Rust, Python vLLM, and Python SGLang frontend preprocessing paths use this setting for `auto`, required, and named tool choices. Setting it to `false` disables optional guidance. Rust preserves native tags for Kimi K2 required/named and Kimi K3 named choices; Python vLLM and SGLang respect the opt-out for all tool choices. Native Rust sidecars retain their conservative `off`/`auto` behavior, including this Kimi exception, and are not included in the default change. + + Structural tags default to enabled with scope `always` and schema mode `auto` for regular workers. Pass `false` to disable optional guidance, or provide a JSON object for advanced options. Rust retains native tags for Kimi K2 required/named and Kimi K3 named choices; Python vLLM and SGLang respect the opt-out. Native Rust sidecars retain their conservative default. See [Structural Tag](../../use-cases/tool-calling-and-reasoning/structural-tag.md) for configuration and backend compatibility limitations. - Environment variable: `DYN_ENABLE_STRUCTURAL_TAG` - - - - Controls when structural tags are activated. `always` activates them for supported `auto`, required, and named tool requests. `auto` preserves the legacy conditional behavior: required or named `tool_choice`, or automatic tool choice when any tool has `strict=true` or `parallel_tool_calls` is false. `tool_choice=none` is unaffected by either setting. Applies when structural tags are active. - - Allowed values: auto always - - Environment variable: `DYN_STRUCTURAL_TAG_SCOPE` - - - - Controls parameter schema strictness inside structural tags. `auto` applies the declared parameter schema unless a tool explicitly sets `strict=false`; an omitted `strict` field is schema-enforced. `strict` applies the declared schema to every tool and overrides `strict=false`. Applies when structural tags are active. - - Allowed values: auto strict - - Environment variable: `DYN_STRUCTURAL_TAG_SCHEMA` + Environment variable: `DYN_STRUCTURAL_TAG` diff --git a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md index a54b02bde23e..acb528d6ce06 100644 --- a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md +++ b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md @@ -19,8 +19,8 @@ Benefits: JSON schema. - **Single-call enforcement** — `parallel_tool_calls=false` is enforced via `stop_after_first` in the grammar, not just by convention. -- **Tool call ban** — when `tool_choice="none"`, specific tokenizer tokens can be - banned so the model cannot start native tool-call syntax (see +- **Tool call ban** — when `tool_choice="none"`, parser-specific strings can be + excluded so the model cannot complete native tool-call syntax (see [trade-offs](#tool_choicenone-and-token-banning)). ## Prerequisites @@ -88,30 +88,59 @@ decoding. See [Activation Scope](#activation-scope) for the exact policy. | Flag | Values | Default | Description | |---|---|---|---| -| `--dyn-enable-structural-tag` | bool | `true` | Enables tool-call structural tags in Rust, Python vLLM, and Python SGLang preprocessing. Rust retains native forced-tool tags for Kimi K2 required/named and Kimi K3 named choices when disabled. | -| `--dyn-structural-tag-scope` | `auto`, `always` | `always` | Controls when structural tags are activated (see [Activation Scope](#activation-scope)). | -| `--dyn-structural-tag-schema` | `auto`, `strict` | `auto` | Controls parameter schema strictness inside structural tags (see [Schema Modes](#schema-modes)). | +| `--dyn-structural-tag` | boolean or JSON object | enabled | Configure structural tags, or pass `false` to disable optional guidance. | + +Custom `--dyn-structural-tag` config example: + +```json +{ + "scope": "always", + "schema": "strict", + "allow_tool_calls_with_structured_output": true, + "exclude_special_tokens": false, + "reasoning_boundary": "backend", + "tool_arguments_any_order": true +} +``` + +The flag without a value uses the defaults below. Every field is optional: + +| Field | Values | Default | Description | +|---|---|---|---| +| `scope` | `auto`, `always` | `always` | Selects eligible tool-calling requests. | +| `schema` | `auto`, `strict` | `auto` | Selects which tool argument schemas are enforced. | +| `allow_tool_calls_with_structured_output` | boolean | `false` | Lets `tool_choice="auto"` choose between tool calls and a schema-constrained final response. Requires parsers v2. | +| `exclude_special_tokens` | boolean, `null` | `null` | Controls reasoning and tool-call marker exclusions. `null` preserves the model-family default. Requires parsers v2. | +| `reasoning_boundary` | `auto`, `structural_tag`, `backend` | `auto` | Selects whether the structural tag closes prompt-opened reasoning or the inference engine activates the post-reasoning grammar. `auto` follows the backend's advertised policy. `backend` requires parsers v2 and a reasoning parser. | +| `tool_arguments_any_order` | boolean | `false` | Allows tool argument properties in any order. This weakens required-property and duplicate-key validation and requires parsers v2 with XGrammar >= 0.2.3. Structured-output schemas are unaffected. | + +If the backend advertises that it owns reasoning-aware grammar activation, +explicitly selecting `structural_tag` is rejected to avoid applying both +reasoning gates. Explicit `backend` selection assumes the inference engine is +configured accordingly. + +`DYN_STRUCTURAL_TAG` accepts `true`, `false`, or the same JSON object. Unknown +fields and invalid values are rejected during worker startup. ## Preserve Previous Behavior -To disable optional structural guidance, pass the negated flag to the worker: +To disable optional structural guidance, pass `false` to the worker: ```bash -python3 -m dynamo.vllm ... --no-dyn-enable-structural-tag -python3 -m dynamo.sglang ... --no-dyn-enable-structural-tag -python3 -m dynamo.trtllm ... --no-dyn-enable-structural-tag +python3 -m dynamo.vllm ... --dyn-structural-tag false +python3 -m dynamo.sglang ... --dyn-structural-tag false +python3 -m dynamo.trtllm ... --dyn-structural-tag false ``` Or set the equivalent environment variable before starting the worker: ```bash -export DYN_ENABLE_STRUCTURAL_TAG=false +export DYN_STRUCTURAL_TAG=false ``` Rust preprocessing still applies native tags to Kimi K2 `required`/named -choices and Kimi K3 named choices when structural tags are disabled. Kimi K3 -named tags use the new argument-schema enforcement; the opt-out does not -restore main's grammar. A lookahead pattern can still be rejected by XGrammar +choices and Kimi K3 named choices when optional structural tags are disabled. +A lookahead pattern can still be rejected by XGrammar 0.2.1 or 0.2.7 unless the tool sets `strict: false` under schema mode `auto`. Global schema mode `strict` overrides that workaround. Kimi K2/K3 automatic choices and Kimi K3 `required` do not use this exception. Python vLLM and SGLang @@ -121,13 +150,17 @@ To keep structural tags enabled but preserve the previous conditional activation policy, set the scope to `auto`: ```bash -export DYN_STRUCTURAL_TAG_SCOPE=auto +export DYN_STRUCTURAL_TAG='{"scope":"auto"}' ``` With this scope, required and named tool choices remain eligible. Automatic tool choice uses structural tags only when a tool sets `strict: true` or the request sets `parallel_tool_calls` to `false`. +The legacy `--dyn-enable-structural-tag`, `--no-dyn-enable-structural-tag`, +`--dyn-structural-tag-scope`, and `--dyn-structural-tag-schema` flags and their +environment variables remain accepted with a deprecation warning. + Setting `strict: false` on a tool relaxes its argument schema. On vLLM 0.30.0, pinned by Dynamo's CUDA image, auto choice with every tool explicitly non-strict also gets no structural tag from the registry; see @@ -147,6 +180,10 @@ Currently tested and supported: - `deepseek_v3_2`, `deepseek_v4` - `kimi_k2`, `kimi_k3`, `kimi-k3` - `inkling` +- `glm47` with parsers v2 + +The parsers-v2 builders for `qwen3_coder`, `deepseek_v4`, and `glm47` require +`DYN_PARSER_VERSION=2` in the worker and frontend processes. Contributions adding structural tag support for new parsers are welcome. @@ -164,7 +201,7 @@ therefore differ by backend and engine version. ## Activation Scope -The `--dyn-structural-tag-scope` flag controls when structural tags are used +The `scope` field controls when structural tags are used based on the request's `tool_choice`: ### `always` (default) @@ -181,11 +218,11 @@ based on the request's `tool_choice`: |---|---| | `required` / `named` | Always | | `auto` | Only when any tool has `strict: true` or `parallel_tool_calls` is `false` | -| `none` | Exclusion tag on the Rust path only (see [below](#tool_choicenone-and-token-banning)) | +| `none` | Exclusion tag only when tools remain in the prompt (see [below](#tool_choicenone-and-token-banning)) | ## Schema Modes -The `--dyn-structural-tag-schema` flag controls what JSON schema is used for +The `schema` field controls what JSON schema is used for tool arguments inside the structural tag: ### `auto` (default) @@ -202,7 +239,7 @@ tool arguments inside the structural tag: If a model-native builder cannot safely represent part of a schema, it keeps the strongest safe tool envelope and relaxes that argument section. If the -builder cannot produce a structural tag at all, Dynamo uses the existing +builder cannot produce an automatically added tool-call tag, Dynamo uses the existing compatibility path rather than introducing a new request error; automatic tool choice may therefore remain unconstrained for that parser/schema combination. @@ -246,19 +283,18 @@ This interacts with the `--exclude-tools-when-tool-choice-none` flag (default: | `exclude-tools-when-tool-choice-none` | Structural tag | Effect | |---|---|---| | `true` (default) | off | Tools removed from prompt. Model doesn't know about tools. Prompt changes break KV cache prefix sharing. | -| `true` | on | Tools removed from prompt; tokens also banned. Prompt changes break KV cache prefix sharing. | -| `false` | on | Tools stay in prompt; guided decoding bans tokens. Model sees tools but cannot emit banned openings. Stable KV cache prefix across different `tool_choice` values. | +| `true` | on | Tools removed from prompt. Prompt changes break KV cache prefix sharing. | +| `false` | on | Tools stay in prompt; guided decoding excludes tool-call markers. Model sees tools but cannot complete a native tool-call opening. Stable KV cache prefix across different `tool_choice` values. | | `false` | off | Tools stay in prompt; no token ban. Same response shaping as above: no structured `tool_calls` for explicit `none`. Tool-like text may still appear in `content`. | For multi-turn conversations where `tool_choice` changes between turns, consider `--no-exclude-tools-when-tool-choice-none` combined with -`--dyn-enable-structural-tag` to keep the prompt stable and benefit from +`--dyn-structural-tag` to keep the prompt stable and benefit from KV cache reuse. ## Example -To pin the enabled state, scope, and schema explicitly, add the following flags -to the worker `args:` alongside the parser: +To pin the scope and schema, pass a JSON value to `--dyn-structural-tag`: ```yaml - name: SGLangWorker @@ -283,11 +319,8 @@ to the worker `args:` alongside the parser: - Qwen/Qwen3.5-4B - --dyn-tool-call-parser - qwen3_coder - - --dyn-enable-structural-tag - - --dyn-structural-tag-scope - - always - - --dyn-structural-tag-schema - - strict + - --dyn-structural-tag + - '{"scope":"always","schema":"strict"}' ``` ## See Also diff --git a/lib/backend-common/src/worker.rs b/lib/backend-common/src/worker.rs index 4290689f907d..85eaa29d6789 100644 --- a/lib/backend-common/src/worker.rs +++ b/lib/backend-common/src/worker.rs @@ -15,8 +15,7 @@ use std::time::Duration; use dynamo_llm::first_token::FirstTokenSource; use dynamo_llm::local_model::runtime_config::{ - DisaggregatedEndpoint, ModelRuntimeConfig, StructuralTagMode, StructuralTagSchemaMode, - StructuralTagScope, TOPOLOGY_TAINT_PREFIX, + DisaggregatedEndpoint, ModelRuntimeConfig, StructuralTagConfig, TOPOLOGY_TAINT_PREFIX, }; use dynamo_llm::local_model::{LocalModel, LocalModelBuilder, update_model_taints}; use dynamo_llm::model_type::{ModelInput, ModelType}; @@ -209,12 +208,8 @@ pub struct WorkerConfig { /// Python sets this via `--health-check-payload` / env; Rust-only /// engines leave it `None` and let `Worker` read the env directly. pub health_check_payload: Option, - /// Structural tag guided decoding mode. - pub structural_tag_mode: StructuralTagMode, - /// Structural tag activation scope. - pub structural_tag_scope: StructuralTagScope, - /// Structural tag schema strictness. - pub structural_tag_schema: StructuralTagSchemaMode, + /// Structural-tag guided-decoding policy. Presence enables structural tags. + pub structural_tag: Option, /// Runtime / transport overrides used when constructing the /// `DistributedRuntime`. pub runtime: RuntimeConfig, @@ -264,12 +259,9 @@ impl Default for WorkerConfig { metrics_labels: Vec::new(), disaggregation_mode: DisaggregationMode::Aggregated, health_check_payload: None, - // Keep the shared/wire default conservative for mixed-version and - // Rust-sidecar compatibility. Deployment-facing runtime arguments - // explicitly publish the current On/Always defaults. - structural_tag_mode: StructuralTagMode::Off, - structural_tag_scope: StructuralTagScope::Auto, - structural_tag_schema: StructuralTagSchemaMode::Auto, + // Native sidecars retain their conservative default; Python workers + // publish the deployment-facing On/Always policy explicitly. + structural_tag: None, runtime: RuntimeConfig::default(), route_to_encoder: false, enable_rl: false, @@ -2211,9 +2203,7 @@ async fn build_local_model( tool_call_parser: config.tool_call_parser.clone(), reasoning_parser: config.reasoning_parser.clone(), exclude_tools_when_tool_choice_none: config.exclude_tools_when_tool_choice_none, - structural_tag_mode: config.structural_tag_mode, - structural_tag_scope: config.structural_tag_scope, - structural_tag_schema: config.structural_tag_schema, + structural_tag: config.structural_tag.clone(), enable_local_indexer, kv_state_endpoint: config.kv_state_endpoint.clone(), disaggregated_endpoint, diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index e40a567e8f74..fbfca5ee064c 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -2319,9 +2319,9 @@ dependencies = [ [[package]] name = "dynamo-parsers-v2" -version = "0.7.17" +version = "0.7.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f80b88ab86928219f3444822dc33c1ca170d0cc67beeb40d18cf7d6f287bad5e" +checksum = "f43e47b6a68353cc04d4dfe19c69414a5884c9963b72c68937d019998c5ef72e" dependencies = [ "anyhow", "openai-harmony", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index 30e520944bc1..7b512840c3db 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2371,9 +2371,9 @@ dependencies = [ [[package]] name = "dynamo-parsers-v2" -version = "0.7.17" +version = "0.7.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f80b88ab86928219f3444822dc33c1ca170d0cc67beeb40d18cf7d6f287bad5e" +checksum = "f43e47b6a68353cc04d4dfe19c69414a5884c9963b72c68937d019998c5ef72e" dependencies = [ "anyhow", "openai-harmony", diff --git a/lib/bindings/python/rust/backend.rs b/lib/bindings/python/rust/backend.rs index d87c608640dd..4043b83c5af9 100644 --- a/lib/bindings/python/rust/backend.rs +++ b/lib/bindings/python/rust/backend.rs @@ -27,10 +27,7 @@ use dynamo_backend_common::{ PreprocessedRequest, RawEngine, RuntimeConfig as RsRuntimeConfig, SnapshotPublisher as RsSnapshotPublisher, Worker as RsWorker, WorkerConfig as RsWorkerConfig, }; -use dynamo_llm::local_model::runtime_config::{ - StructuralTagMode as RsStructuralTagMode, StructuralTagSchemaMode as RsStructuralTagSchemaMode, - StructuralTagScope as RsStructuralTagScope, -}; +use dynamo_llm::local_model::runtime_config::StructuralTagConfig as RsStructuralTagConfig; use dynamo_llm::model_type::ModelInput as RsModelInput; use dynamo_runtime as rs; use dynamo_runtime::logging::{DistributedTraceContext, get_distributed_tracing_context}; @@ -434,9 +431,7 @@ impl WorkerConfig { runtime = None, disaggregation_mode = DisaggregationMode::Aggregated, health_check_payload = None, - structural_tag_mode = "off".to_string(), - structural_tag_scope = "auto".to_string(), - structural_tag_schema = "auto".to_string(), + structural_tag = None, route_to_encoder = false, media_decoder = None, media_fetcher = None, @@ -463,9 +458,7 @@ impl WorkerConfig { runtime: Option, disaggregation_mode: DisaggregationMode, health_check_payload: Option, - structural_tag_mode: String, - structural_tag_scope: String, - structural_tag_schema: String, + structural_tag: Option<&Bound<'_, PyDict>>, route_to_encoder: bool, media_decoder: Option, media_fetcher: Option, @@ -498,33 +491,17 @@ impl WorkerConfig { } _ => None, }; - let st_mode = match structural_tag_mode.as_str() { - "off" => RsStructuralTagMode::Off, - "on" => RsStructuralTagMode::On, - other => { - return Err(PyErr::new::(format!( - "Invalid structural_tag_mode: {other}. Expected 'off' or 'on'." - ))); - } - }; - let st_scope = match structural_tag_scope.as_str() { - "auto" => RsStructuralTagScope::Auto, - "always" => RsStructuralTagScope::Always, - other => { - return Err(PyErr::new::(format!( - "Invalid structural_tag_scope: {other}. Expected 'auto' or 'always'." - ))); - } - }; - let st_schema = match structural_tag_schema.as_str() { - "auto" => RsStructuralTagSchemaMode::Auto, - "strict" => RsStructuralTagSchemaMode::Strict, - other => { - return Err(PyErr::new::(format!( - "Invalid structural_tag_schema: {other}. Expected 'auto' or 'strict'." - ))); - } - }; + let structural_tag = structural_tag + .map(|config| -> anyhow::Result<_> { + let value = depythonize(config)?; + Ok(RsStructuralTagConfig::from_value_strict(value)?) + }) + .transpose() + .map_err(|error| { + pyo3::exceptions::PyValueError::new_err(format!( + "Invalid structural_tag config: {error}" + )) + })?; Ok(Self { inner: RsWorkerConfig { namespace, @@ -547,9 +524,7 @@ impl WorkerConfig { metrics_labels, disaggregation_mode: disaggregation_mode.into(), health_check_payload, - structural_tag_mode: st_mode, - structural_tag_scope: st_scope, - structural_tag_schema: st_schema, + structural_tag, runtime: runtime.map(|r| r.inner).unwrap_or_default(), route_to_encoder, // Python vLLM owns and serves its existing `.rl` endpoint. diff --git a/lib/bindings/python/rust/llm/local_model.rs b/lib/bindings/python/rust/llm/local_model.rs index 1b7a5317cda4..33d6bc01a0d3 100644 --- a/lib/bindings/python/rust/llm/local_model.rs +++ b/lib/bindings/python/rust/llm/local_model.rs @@ -10,9 +10,7 @@ use dynamo_kv_router::protocols::{ use dynamo_runtime::protocols::EndpointId; use llm_rs::local_model::runtime_config::DisaggregatedEndpoint as RsDisaggregatedEndpoint; use llm_rs::local_model::runtime_config::ModelRuntimeConfig as RsModelRuntimeConfig; -use llm_rs::local_model::runtime_config::StructuralTagMode as RsStructuralTagMode; -use llm_rs::local_model::runtime_config::StructuralTagSchemaMode as RsStructuralTagSchemaMode; -use llm_rs::local_model::runtime_config::StructuralTagScope as RsStructuralTagScope; +use llm_rs::local_model::runtime_config::StructuralTagConfig as RsStructuralTagConfig; use llm_rs::local_model::runtime_config::TokenizerBackend as RsTokenizerBackend; use llm_rs::protocols::tensor::TensorModelConfig; use pyo3::exceptions::PyValueError; @@ -283,6 +281,19 @@ impl ModelRuntimeConfig { self.inner.exclude_tools_when_tool_choice_none } + #[getter] + fn structural_tag(&self, py: Python<'_>) -> PyResult> { + self.inner + .structural_tag + .as_ref() + .map(|config| { + pythonize::pythonize(py, config) + .map(|value| value.unbind()) + .map_err(to_pyerr) + }) + .transpose() + } + #[getter] fn runtime_data(&self, py: Python<'_>) -> PyResult { let dict = PyDict::new(py); @@ -296,44 +307,18 @@ impl ModelRuntimeConfig { self.inner.get_engine_specific(key).map_err(to_pyerr) } - fn set_structural_tag_mode(&mut self, mode: &str) -> PyResult<()> { - self.inner.structural_tag_mode = match mode { - "off" => RsStructuralTagMode::Off, - "on" => RsStructuralTagMode::On, - _ => { - return Err(PyErr::new::(format!( - "Invalid structural_tag_mode: {mode}. Expected 'off' or 'on'." - ))); - } - }; - Ok(()) - } - - /// Set the structural tag scope ("auto" or "always"). - fn set_structural_tag_scope(&mut self, scope: &str) -> PyResult<()> { - self.inner.structural_tag_scope = match scope { - "auto" => RsStructuralTagScope::Auto, - "always" => RsStructuralTagScope::Always, - _ => { - return Err(PyErr::new::(format!( - "Invalid structural_tag_scope: {scope}. Expected 'auto' or 'always'." - ))); - } - }; - Ok(()) - } - - /// Set the structural tag schema mode ("auto" or "strict"). - fn set_structural_tag_schema(&mut self, schema: &str) -> PyResult<()> { - self.inner.structural_tag_schema = match schema { - "auto" => RsStructuralTagSchemaMode::Auto, - "strict" => RsStructuralTagSchemaMode::Strict, - _ => { - return Err(PyErr::new::(format!( - "Invalid structural_tag_schema: {schema}. Expected 'auto' or 'strict'." - ))); - } - }; + #[pyo3(signature = (structural_tag=None))] + fn set_structural_tag(&mut self, structural_tag: Option<&Bound<'_, PyDict>>) -> PyResult<()> { + let structural_tag = structural_tag + .map(|config| -> anyhow::Result<_> { + let value = pythonize::depythonize(config)?; + Ok(RsStructuralTagConfig::from_value_strict(value)?) + }) + .transpose() + .map_err(|error| { + PyValueError::new_err(format!("Invalid structural_tag config: {error}")) + })?; + self.inner.structural_tag = structural_tag; Ok(()) } diff --git a/lib/bindings/python/src/dynamo/_core.pyi b/lib/bindings/python/src/dynamo/_core.pyi index 35a891c8c7e2..c5401598a1f7 100644 --- a/lib/bindings/python/src/dynamo/_core.pyi +++ b/lib/bindings/python/src/dynamo/_core.pyi @@ -895,6 +895,7 @@ class ModelRuntimeConfig: tool_call_parser: str | None reasoning_parser: str | None tokenizer_backend: str | None + structural_tag: dict[str, Any] | None exclude_tools_when_tool_choice_none: bool data_parallel_start_rank: int data_parallel_size: int @@ -923,16 +924,8 @@ class ModelRuntimeConfig: """Get an engine-specific runtime configuration value""" ... - def set_structural_tag_mode(self, mode: str) -> None: - """Set structural tag mode ("off" or "on").""" - ... - - def set_structural_tag_scope(self, scope: str) -> None: - """Set structural tag scope ("auto" or "always").""" - ... - - def set_structural_tag_schema(self, schema: str) -> None: - """Set structural tag schema mode ("auto" or "strict").""" + def set_structural_tag(self, structural_tag: dict[str, Any] | None) -> None: + """Set the structural-tag policy, or disable it with ``None``.""" ... def set_disaggregated_endpoint( @@ -3339,9 +3332,7 @@ class backend: runtime: Optional["backend.RuntimeConfig"] = None, disaggregation_mode: "backend.DisaggregationMode" = ..., health_check_payload: Optional[Dict[str, Any]] = None, - structural_tag_mode: str = ..., - structural_tag_scope: str = ..., - structural_tag_schema: str = ..., + structural_tag: Optional[Dict[str, Any]] = None, route_to_encoder: bool = ..., media_decoder: Optional[MediaDecoder] = None, media_fetcher: Optional[MediaFetcher] = None, diff --git a/lib/llm/src/discovery/watcher.rs b/lib/llm/src/discovery/watcher.rs index 7d369fe175db..dca839d832a7 100644 --- a/lib/llm/src/discovery/watcher.rs +++ b/lib/llm/src/discovery/watcher.rs @@ -1043,6 +1043,7 @@ impl ControllerHost for ModelWatcher { let mut card = instance.deserialize_model::()?; normalize_legacy_prefill_topology(&mut card); + card.runtime_config.normalize_structural_tag_compatibility(); self.apply_tokenizer_overrides(&mut card); validate_card_shape(&card)?; anyhow::ensure!( @@ -3226,6 +3227,64 @@ request_classifier: assert_eq!(legacy.mdc_checksum, current.mdc_checksum); } + #[tokio::test] + async fn discovery_normalizes_legacy_and_future_structural_tag_fields() { + use crate::local_model::runtime_config::{StructuralTagConfig, StructuralTagScope}; + + let runtime = Runtime::from_current().unwrap(); + let drt = DistributedRuntime::new(runtime, DistributedConfig::process_local()) + .await + .unwrap(); + let watcher = ModelWatcher::new( + drt, + Arc::new(ModelManager::new()), + RouterConfig::default(), + 0, + None, + None, + None, + Arc::new(Metrics::new()), + ); + + let card = serde_json::to_value(ModelDeploymentCard::with_name_only("model")).unwrap(); + let mut legacy_card = card.clone(); + let legacy_runtime = legacy_card["runtime_config"].as_object_mut().unwrap(); + legacy_runtime.insert("structural_tag_mode".into(), serde_json::json!("on")); + legacy_runtime.insert("structural_tag_scope".into(), serde_json::json!("always")); + + let mut future_card = card; + future_card["runtime_config"]["structural_tag"] = serde_json::json!({ + "scope": "always", + "future_option": true + }); + + let normalize = |instance_id, card_json| { + watcher + .normalize( + DiscoveryInstance::Model { + namespace: "ns1".to_string(), + component: "workers".to_string(), + endpoint: "generate".to_string(), + instance_id, + card_json, + model_suffix: None, + }, + &NamespaceFilter::Global, + ) + .unwrap() + .unwrap() + }; + let legacy = normalize(1, legacy_card); + let future = normalize(2, future_card); + let expected = Some(StructuralTagConfig { + scope: StructuralTagScope::Always, + ..Default::default() + }); + + assert_eq!(legacy.card.runtime_config.structural_tag, expected); + assert_eq!(future.card.runtime_config.structural_tag, expected); + } + #[test] fn ws_key_separates_prefill_from_decode_in_same_namespace() { let endpoint_id = test_endpoint_id("generate"); diff --git a/lib/llm/src/discovery/worker_set.rs b/lib/llm/src/discovery/worker_set.rs index 29996b6cd39b..9f93f6900bd3 100644 --- a/lib/llm/src/discovery/worker_set.rs +++ b/lib/llm/src/discovery/worker_set.rs @@ -425,8 +425,7 @@ impl WorkerSet { .get(crate::preprocessor::DEFAULT_THINKING_MODE_RUNTIME_KEY) .and_then(serde_json::Value::as_str) .map(str::to_string), - structural_tag_mode: self.card.runtime_config.structural_tag_mode, - structural_tag_scope: self.card.runtime_config.structural_tag_scope, + structural_tag: self.card.runtime_config.structural_tag.clone(), exclude_tools_when_tool_choice_none: self .card .runtime_config diff --git a/lib/llm/src/http/service.rs b/lib/llm/src/http/service.rs index 45ecef2f637f..7baaa492c4c8 100644 --- a/lib/llm/src/http/service.rs +++ b/lib/llm/src/http/service.rs @@ -78,13 +78,19 @@ fn apply_request_tool_call_parsing_options( let converted_tool_choice = crate::preprocessor::tool_choice::convert_tool_choice(tool_choice); let effective_tools = crate::preprocessor::tool_choice::effective_tools(&request.inner)?; let converted_tools = crate::preprocessor::tool_choice::convert_tools(effective_tools.as_ref()); + let structural_tag_config = parsing_options.structural_tag.as_ref(); + let has_structured_output = request + .inner + .response_format + .as_ref() + .is_some_and(|format| !matches!(format, dynamo_protocols::types::ResponseFormat::Text)); let uses_structural_tag = crate::preprocessor::structural_tag::structural_tag_decision( parsing_options.tool_call_parser.as_deref(), &converted_tool_choice, &converted_tools, request.inner.parallel_tool_calls, - parsing_options.structural_tag_mode, - parsing_options.structural_tag_scope, + structural_tag_config, + has_structured_output, parsing_options.exclude_tools_when_tool_choice_none, )? .is_required(); @@ -402,15 +408,14 @@ mod tests { // The registry-builder gap this test suite exists to close: a non-Kimi parser // (qwen3_coder, which the parser registry does register a structural-tag builder // for) with a forced tool_choice must still resolve to `StructuralTag` once the - // operator has globally enabled structural-tag mode — matching the real - // preprocessing path (`apply_tool_choice_structural_tag`'s - // `structural_tag_mode != Off` branch), not the narrower Kimi-only reconstruction - // this helper used before this fix. + // operator has configured structural tags — matching the real preprocessing + // path, not the narrower Kimi-only reconstruction this helper used before + // this fix. #[test] fn operator_enabled_mode_resolves_structural_tag_for_a_non_kimi_parser() { let parsing_options = ParsingOptions { tool_call_parser: Some("qwen3_coder".to_string()), - structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::On, + structural_tag: Some(Default::default()), ..Default::default() }; let result = @@ -419,15 +424,14 @@ mod tests { assert_eq!( result.guided_tool_constraint, GuidedToolConstraint::StructuralTag, - "an operator-enabled structural_tag_mode must apply to any registry-supported \ + "an operator-configured structural tag must apply to any registry-supported \ parser, not only the Kimi-intrinsic case" ); } - // With the low-level ParsingOptions default (`structural_tag_mode = Off`), - // the same non-Kimi parser must NOT get a structural tag. Regular workers - // explicitly publish the deployment's default-on policy into the model card; - // native sidecars that do not do so retain this conservative default. + // The low-level ParsingOptions default remains conservative. Regular + // workers explicitly publish the deployment-facing On/Always policy; + // native sidecars without an operator configuration retain optional tags OFF. #[test] fn parsing_options_default_mode_off_does_not_resolve_structural_tag() { let parsing_options = ParsingOptions { @@ -443,8 +447,40 @@ mod tests { ); } + #[test] + fn tools_with_structured_output_are_classified_as_structural_tag() { + let mut request = request(json!("auto")); + request.inner.response_format = Some( + serde_json::from_value(json!({ + "type": "json_schema", + "json_schema": { + "name": "response", + "schema": {"type": "object"} + } + })) + .expect("response_format must deserialize"), + ); + let config = crate::local_model::runtime_config::StructuralTagConfig { + allow_tool_calls_with_structured_output: true, + ..Default::default() + }; + let parsing_options = ParsingOptions { + tool_call_parser: Some("qwen3_coder".to_string()), + structural_tag: Some(config), + ..Default::default() + }; + + let result = apply_request_tool_call_parsing_options(parsing_options, &request) + .expect("tools with structured output must resolve a constraint"); + + assert_eq!( + result.guided_tool_constraint, + GuidedToolConstraint::StructuralTag + ); + } + // `tool_choice: none` must never leave the client able to observe a tool call or - // a tool-call finish reason, regardless of the operator's structural-tag mode or + // a tool-call finish reason, regardless of the operator's structural-tag policy or // exclusion setting. The real preprocessing path may install a ban tag at // generation time for `none` (which is a request-time generation constraint on // the engine, tracked separately from this field), but that must not surface as @@ -459,7 +495,7 @@ mod tests { for exclude_tools_when_tool_choice_none in [true, false] { let parsing_options = ParsingOptions { tool_call_parser: Some("qwen3_coder".to_string()), - structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::On, + structural_tag: Some(Default::default()), exclude_tools_when_tool_choice_none, ..Default::default() }; @@ -659,7 +695,7 @@ mod tests { } // Same gap, operator-enabled path: qwen3_coder only gets a structural tag when - // `structural_tag_mode = On` (it is not Kimi-intrinsic). Confirms the fix isn't + // structural tags are configured (it is not Kimi-intrinsic). Confirms the fix isn't // narrowly scoped to the two Kimi-intrinsic parsers. #[test] fn operator_enabled_qwen3_coder_required_with_empty_tools_is_rejected() { @@ -673,7 +709,7 @@ mod tests { serde_json::from_value(value).expect("request must deserialize"); let parsing_options = ParsingOptions { tool_call_parser: Some("qwen3_coder".to_string()), - structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::On, + structural_tag: Some(Default::default()), ..Default::default() }; let result = apply_request_tool_call_parsing_options(parsing_options, &request); @@ -706,7 +742,7 @@ mod tests { serde_json::from_value(value).expect("request must deserialize"); let parsing_options = ParsingOptions { tool_call_parser: Some("qwen3_coder".to_string()), - structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::On, + structural_tag: Some(Default::default()), ..Default::default() }; let result = apply_request_tool_call_parsing_options(parsing_options, &request); diff --git a/lib/llm/src/local_model/runtime_config.rs b/lib/llm/src/local_model/runtime_config.rs index 3008f6f58099..d33c01721e30 100644 --- a/lib/llm/src/local_model/runtime_config.rs +++ b/lib/llm/src/local_model/runtime_config.rs @@ -69,7 +69,16 @@ pub fn topology_taint(domain: &str, value: &str) -> String { format!("{TOPOLOGY_TAINT_PREFIX}{domain}={value}") } -/// Master switch for structural tag guided decoding. +/// Controls when structural tags are activated based on `tool_choice`. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(rename_all = "snake_case")] +pub enum StructuralTagScope { + #[default] + Auto, + Always, +} + +/// Legacy structural-tag enablement field used by older frontends. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "snake_case")] pub enum StructuralTagMode { @@ -78,13 +87,54 @@ pub enum StructuralTagMode { On, } -/// Controls when structural tags are activated based on `tool_choice`. +/// Controls which layer owns a reasoning boundary before structural-tag output. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "snake_case")] -pub enum StructuralTagScope { +pub enum StructuralTagReasoningBoundary { + /// Follow the reasoning-boundary policy advertised by the inference backend. #[default] Auto, - Always, + /// Include reasoning and its closing marker in the structural tag. + StructuralTag, + /// Let the inference backend activate a suffix-only structural tag after reasoning. + Backend, +} + +/// Structural-tag guided-decoding policy for tool-calling requests. +/// +/// Presence enables structural tags; absence disables operator-controlled tags. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +#[serde(default)] +pub struct StructuralTagConfig { + pub scope: StructuralTagScope, + pub schema: StructuralTagSchemaMode, + pub allow_tool_calls_with_structured_output: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub exclude_special_tokens: Option, + pub reasoning_boundary: StructuralTagReasoningBoundary, + pub tool_arguments_any_order: bool, +} + +impl StructuralTagConfig { + pub fn from_value_strict(value: serde_json::Value) -> serde_json::Result { + const FIELDS: &[&str] = &[ + "scope", + "schema", + "allow_tool_calls_with_structured_output", + "exclude_special_tokens", + "reasoning_boundary", + "tool_arguments_any_order", + ]; + + if let Some(object) = value.as_object() { + for field in object.keys() { + if !FIELDS.contains(&field.as_str()) { + return Err(serde::de::Error::unknown_field(field, FIELDS)); + } + } + } + serde_json::from_value(value) + } } pub const ENV_TOKENIZER_BACKEND: &str = "DYN_TOKENIZER"; @@ -254,15 +304,17 @@ pub struct ModelRuntimeConfig { #[serde(default, skip_serializing_if = "Option::is_none")] pub tokenizer_fallback_enabled: Option, - /// Whether structural tag guided decoding is enabled for tool calls. + /// Structural-tag policy used by the preprocessor. Presence enables + /// operator-controlled structural tags. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub structural_tag: Option, + + // Compatibility with v1.4/v1.5 workers and frontends during v1.6 rolling upgrades + // TODO: remove when v1.5 falls outside the N-2 compatibility window #[serde(default)] pub structural_tag_mode: StructuralTagMode, - - /// Controls when structural tags are activated based on tool_choice. #[serde(default)] pub structural_tag_scope: StructuralTagScope, - - /// Controls whether tools get real or generic schemas in structural tags. #[serde(default)] pub structural_tag_schema: StructuralTagSchemaMode, @@ -396,8 +448,9 @@ impl Default for ModelRuntimeConfig { tool_call_arguments_format: ToolCallArgumentsFormat::JsonString, tokenizer_backend: None, tokenizer_fallback_enabled: None, - // Missing fields from older workers remain conservative. Current - // deployment configuration explicitly publishes On/Always. + // Older cards and native sidecars remain conservative. Current + // Python workers explicitly publish the deployment-facing policy. + structural_tag: None, structural_tag_mode: StructuralTagMode::Off, structural_tag_scope: StructuralTagScope::Auto, structural_tag_schema: StructuralTagSchemaMode::Auto, @@ -423,6 +476,27 @@ impl Default for ModelRuntimeConfig { } impl ModelRuntimeConfig { + /// Normalize older wire fields into the current policy and project it back for older readers + pub(crate) fn normalize_structural_tag_compatibility(&mut self) { + if self.structural_tag.is_none() && self.structural_tag_mode == StructuralTagMode::On { + self.structural_tag = Some(StructuralTagConfig { + scope: self.structural_tag_scope, + schema: self.structural_tag_schema, + ..Default::default() + }); + } + + if let Some(config) = &self.structural_tag { + self.structural_tag_mode = StructuralTagMode::On; + self.structural_tag_scope = config.scope; + self.structural_tag_schema = config.schema; + } else { + self.structural_tag_mode = StructuralTagMode::Off; + self.structural_tag_scope = StructuralTagScope::Auto; + self.structural_tag_schema = StructuralTagSchemaMode::Auto; + } + } + /// Check whether a runtime boolean is explicitly enabled. /// /// Rust callers commonly store booleans, while compatibility cards may @@ -806,6 +880,85 @@ mod tests { use crate::protocols::openai::chat_completions::tool_parser_v2::V2_FAMILIES; + #[test] + fn structural_tag_config_accepts_future_wire_fields_and_rejects_unknown_input_fields() { + assert_eq!( + serde_json::from_value::(serde_json::json!({})).unwrap(), + StructuralTagConfig::default() + ); + assert_eq!( + StructuralTagConfig::default().reasoning_boundary, + StructuralTagReasoningBoundary::Auto + ); + + let config = StructuralTagConfig { + scope: StructuralTagScope::Always, + schema: StructuralTagSchemaMode::Strict, + allow_tool_calls_with_structured_output: true, + exclude_special_tokens: Some(false), + reasoning_boundary: StructuralTagReasoningBoundary::Backend, + tool_arguments_any_order: true, + }; + let value = serde_json::to_value(&config).unwrap(); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + config + ); + assert_eq!( + StructuralTagConfig::from_value_strict(value).unwrap(), + config + ); + + let future_config = serde_json::json!({"future_option": true}); + assert_eq!( + serde_json::from_value::(future_config.clone()).unwrap(), + StructuralTagConfig::default() + ); + let error = StructuralTagConfig::from_value_strict(future_config).unwrap_err(); + assert!(error.to_string().contains("unknown field `future_option`")); + } + + #[test] + fn legacy_structural_tag_fields_normalize_to_current_config() { + let mut runtime: ModelRuntimeConfig = serde_json::from_value(serde_json::json!({ + "structural_tag_mode": "on", + "structural_tag_scope": "always", + "structural_tag_schema": "strict" + })) + .unwrap(); + + runtime.normalize_structural_tag_compatibility(); + + assert_eq!( + runtime.structural_tag, + Some(StructuralTagConfig { + scope: StructuralTagScope::Always, + schema: StructuralTagSchemaMode::Strict, + ..Default::default() + }) + ); + } + + #[test] + fn current_structural_tag_config_controls_legacy_projection() { + let mut runtime: ModelRuntimeConfig = serde_json::from_value(serde_json::json!({ + "structural_tag": {"scope": "auto", "schema": "strict"}, + "structural_tag_mode": "off", + "structural_tag_scope": "always", + "structural_tag_schema": "auto" + })) + .unwrap(); + + runtime.normalize_structural_tag_compatibility(); + + let wire = serde_json::to_value(&runtime).unwrap(); + assert_eq!(wire["structural_tag_mode"], "on"); + assert_eq!(wire["structural_tag_scope"], "auto"); + assert_eq!(wire["structural_tag_schema"], "strict"); + assert_eq!(wire["structural_tag"]["scope"], "auto"); + assert_eq!(wire["structural_tag"]["schema"], "strict"); + } + // Env-touching tests use `temp_env` (snapshot + restore around the closure) and // `#[serial_test::serial]` (serialize against every other env-touching test in the // binary, not just this module). A module-local mutex would be insufficient because diff --git a/lib/llm/src/model_card.rs b/lib/llm/src/model_card.rs index 99f27b7231d3..dcb5e5fa9f7e 100644 --- a/lib/llm/src/model_card.rs +++ b/lib/llm/src/model_card.rs @@ -1159,6 +1159,7 @@ impl ModelDeploymentCard { pub(crate) fn for_mdc_wire(&self) -> Self { let mut card = self.clone(); card.legacy_context_length = Some(self.effective_context_length()); + card.runtime_config.normalize_structural_tag_compatibility(); card } @@ -3181,6 +3182,9 @@ mod tests { #[cfg(test)] mod ownership_tests { use super::*; + use crate::local_model::runtime_config::{ + StructuralTagConfig, StructuralTagSchemaMode, StructuralTagScope, + }; #[test] fn architectural_context_prefers_config_then_text_config_then_tokenizer() -> anyhow::Result<()> @@ -3386,6 +3390,24 @@ mod ownership_tests { assert_eq!(wire_value["context_length"], 8_192); } + #[test] + fn structural_tag_wire_includes_fields_for_older_frontends() { + let mut card = ModelDeploymentCard::with_name_only("model"); + card.runtime_config.structural_tag = Some(StructuralTagConfig { + scope: StructuralTagScope::Always, + schema: StructuralTagSchemaMode::Strict, + ..Default::default() + }); + + let wire: serde_json::Value = serde_json::from_str(&card.to_json().unwrap()).unwrap(); + let runtime = &wire["runtime_config"]; + assert_eq!(runtime["structural_tag_mode"], "on"); + assert_eq!(runtime["structural_tag_scope"], "always"); + assert_eq!(runtime["structural_tag_schema"], "strict"); + assert_eq!(runtime["structural_tag"]["scope"], "always"); + assert_eq!(runtime["structural_tag"]["schema"], "strict"); + } + #[test] fn tensor_config_serializes_at_card_top_level() { let mut card = ModelDeploymentCard::with_name_only("tensor"); diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 994cd377daf2..30f623f8da4c 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -1706,6 +1706,7 @@ pub struct OpenAIPreprocessor { lora_name: Option, /// Per-model runtime configuration propagated to response generator (e.g., reasoning/tool parser) runtime_config: crate::local_model::runtime_config::ModelRuntimeConfig, + structural_tag_reasoning_boundary: structural_tag::ResolvedReasoningBoundary, /// KV cache block size published in the model deployment card. kv_cache_block_size: usize, tool_call_parser: Option, @@ -2582,6 +2583,8 @@ impl OpenAIPreprocessor { // // Initialize runtime config from the ModelDeploymentCard let runtime_config = mdc.runtime_config.clone(); + let structural_tag_reasoning_boundary = + structural_tag::validate_runtime_config(&runtime_config)?; let token_budget = match runtime_config .get_engine_specific::(TOKEN_BUDGET_RUNTIME_KEY) { @@ -2894,6 +2897,7 @@ impl OpenAIPreprocessor { mdcsum, lora_name, runtime_config, + structural_tag_reasoning_boundary, kv_cache_block_size, tool_call_parser, parser_requires_special_tokens, @@ -5154,11 +5158,11 @@ impl OpenAIPreprocessor { && tool_parser_v2::enabled() && selected_version == tool_parser_v2::ParserVersion::V2 && tool_parser_v2::supports_family(parser_name) - && !uses_tool_call_structural_tag - && matches!( - request.inner.tool_choice.as_ref(), - None | Some(ChatCompletionToolChoiceOption::Auto) - ) + && (uses_tool_call_structural_tag + || matches!( + request.inner.tool_choice.as_ref(), + None | Some(ChatCompletionToolChoiceOption::Auto) + )) { let parser_name = match parser_name { "deepseek-v4" | "deepseekv4" => "deepseek_v4", diff --git a/lib/llm/src/preprocessor/structural_tag.rs b/lib/llm/src/preprocessor/structural_tag.rs index 45bca09450c6..d3228acc7cb4 100644 --- a/lib/llm/src/preprocessor/structural_tag.rs +++ b/lib/llm/src/preprocessor/structural_tag.rs @@ -3,16 +3,87 @@ //! Structural tag policy for chat tool-call guided decoding. +mod v1; +mod v2; + +use std::borrow::Cow; + use crate::local_model::runtime_config::{ - StructuralTagMode, StructuralTagScope, TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + ModelRuntimeConfig, StructuralTagConfig, StructuralTagReasoningBoundary, StructuralTagScope, + TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, }; use crate::preprocessor::{OpenAIPreprocessor, PreprocessedRequest}; use crate::protocols::openai::tools::{ToolChoiceValidation, validate_tool_choice_against_names}; -use dynamo_parsers::tool_calling::{ToolChoice, ToolDefinition}; +use dynamo_parsers::tool_calling::{StructuralTagSchemaMode, ToolChoice, ToolDefinition}; +use dynamo_protocols::types::ResponseFormat; +use dynamo_runtime::config::environment_names::llm as env_llm; use dynamo_runtime::error::{DynamoError, ErrorType}; +struct StructuralTagBuildRequest<'a> { + tool_choice: &'a ToolChoice, + tools: &'a [ToolDefinition], + parallel_tool_calls: Option, + schema_mode: StructuralTagSchemaMode, + exclude_special_tokens: Option, + reasoning_boundary: ResolvedReasoningBoundary, + tool_arguments_any_order: bool, + starts_in_reasoning: bool, + structured_output_schema: Option<&'a serde_json::Value>, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ResolvedReasoningBoundary { + StructuralTag, + Backend, +} + +fn resolve_reasoning_boundary( + configured: StructuralTagReasoningBoundary, + backend_excludes_reasoning: bool, +) -> Result { + match (configured, backend_excludes_reasoning) { + (StructuralTagReasoningBoundary::Auto, false) + | (StructuralTagReasoningBoundary::StructuralTag, false) => { + Ok(ResolvedReasoningBoundary::StructuralTag) + } + (StructuralTagReasoningBoundary::Auto, true) + | (StructuralTagReasoningBoundary::Backend, _) => Ok(ResolvedReasoningBoundary::Backend), + (StructuralTagReasoningBoundary::StructuralTag, true) => Err( + "structural_tag.reasoning_boundary=structural_tag conflicts with the backend's reasoning-aware guided-decoding policy; use 'auto' or 'backend'", + ), + } +} + +#[derive(Clone, Copy)] +pub(crate) enum SelectedStructuralTagBuilder { + V1(v1::StructuralTagBuilder), + V2(v2::StructuralTagBuilder), +} + +impl SelectedStructuralTagBuilder { + fn for_parser(parser_name: &str) -> Option { + if v2::enabled() + && let Some(builder) = v2::StructuralTagBuilder::for_parser(parser_name) + { + return Some(Self::V2(builder)); + } + + v1::StructuralTagBuilder::for_parser(parser_name).map(Self::V1) + } + + fn build( + self, + request: &StructuralTagBuildRequest<'_>, + ) -> anyhow::Result> { + match self { + Self::V1(builder) => builder.build(request), + Self::V2(builder) => builder.build(request), + } + } +} + /// Validate a forced `tool_choice` against the request's actual `tools` list. /// /// `Required` with no tools, and a `Named` choice for a tool absent from `tools`, @@ -47,8 +118,11 @@ fn is_kimi_k2_parser(parser_name: Option<&str>) -> bool { } fn requires_intrinsic_structural_tag(parser_name: Option<&str>, tool_choice: &ToolChoice) -> bool { - // Preserve native forced-tool grammars even when optional guidance is off. - // K3 required retains its existing prompt/parser path. + // K2 forced calls and K3 named calls cannot use Dynamo's generic JSON-schema + // fallback because both families emit native, marker-delimited formats. + // Treat their structural tags as part of implementing these standard OpenAI + // request shapes, not as an operator opt-in. K3 required remains on its + // intentional prompt-level XTML path. (is_kimi_k2_parser(parser_name) && matches!(tool_choice, ToolChoice::Required | ToolChoice::Named(_))) || (is_kimi_k3_parser(parser_name) && matches!(tool_choice, ToolChoice::Named(_))) @@ -75,19 +149,20 @@ fn should_skip_tool_call_ban(exclude_tools_when_none: bool, tool_choice: &ToolCh /// parser registry can actually build one. /// /// This is the single owner of "is a structural tag applicable and available" — -/// covering the operator's global `structural_tag_mode`/`structural_tag_scope`, -/// intrinsic Kimi forced-tool grammars, the `tool_choice=None` ban-tag exclusion, and real -/// parser-registry builder availability. A caller can only -/// reach `Required` by way of a real, registered -/// [`dynamo_parsers::tool_calling::StructuralTagBuilder`] — it cannot ask for the -/// eligibility half of this decision without also getting the registry-availability -/// proof in the same value. Both the real preprocessing path +/// covering model-family/tool-choice intrinsic eligibility, the configured +/// structural-tag policy, tools-with-structured-output composition, the +/// `tool_choice=None` ban-tag exclusion, and real parser-registry builder +/// availability. A caller can only reach `Required` with a registered builder +/// selected for the active parser generation — it cannot ask for the eligibility +/// half of this decision without also getting the registry-availability proof in +/// the same value. Both the real +/// preprocessing path /// (`apply_tool_choice_structural_tag`) and the HTTP-layer reconstruction /// (`http::service::apply_request_tool_call_parsing_options`) must consult this /// function; neither may re-derive eligibility, mode, scope, or registry /// availability independently. pub(crate) enum StructuralTagDecision { - Required(&'static dynamo_parsers::tool_calling::StructuralTagBuilder), + Required(SelectedStructuralTagBuilder), NotApplicable, } @@ -112,20 +187,36 @@ pub(crate) fn structural_tag_decision( tool_choice: &ToolChoice, tools: &[ToolDefinition], parallel_tool_calls: Option, - mode: StructuralTagMode, - scope: StructuralTagScope, + config: Option<&StructuralTagConfig>, + has_structured_output: bool, exclude_tools_when_tool_choice_none: bool, ) -> Result { - // Validate before any policy gate. Kimi K3 required requests use a + // Validate before any mode or family gate. Kimi K3 required requests use a // prompt-level XTML path, but they still need an actual tool to require. validate_forced_tool_choice(tool_choice, tools)?; - if mode == StructuralTagMode::Off - && !requires_intrinsic_structural_tag(parser_name, tool_choice) - { + if config.is_none() && !requires_intrinsic_structural_tag(parser_name, tool_choice) { return Ok(StructuralTagDecision::NotApplicable); } + let requires_tool_calls_with_structured_output = if has_structured_output { + match tool_choice { + ToolChoice::Auto => { + if config.is_some_and(|config| config.allow_tool_calls_with_structured_output) + && !tools.is_empty() + { + true + } else { + return Ok(StructuralTagDecision::NotApplicable); + } + } + ToolChoice::None => return Ok(StructuralTagDecision::NotApplicable), + ToolChoice::Required | ToolChoice::Named(_) => false, + } + } else { + false + }; + if should_skip_tool_call_ban(exclude_tools_when_tool_choice_none, tool_choice) { // The prompt formatter already omits tools for this request. Avoid // sending a redundant AnyTokens structural tag: vLLM cannot @@ -141,7 +232,7 @@ pub(crate) fn structural_tag_decision( return Ok(StructuralTagDecision::NotApplicable); }; - let Some(builder) = OpenAIPreprocessor::structural_tag_builder_for_parser(parser_name) else { + let Some(builder) = SelectedStructuralTagBuilder::for_parser(parser_name) else { return Ok(StructuralTagDecision::NotApplicable); }; @@ -152,6 +243,12 @@ pub(crate) fn structural_tag_decision( return Ok(StructuralTagDecision::Required(builder)); } + if requires_tool_calls_with_structured_output { + return Ok(StructuralTagDecision::Required(builder)); + } + + let scope = config.map_or_else(Default::default, |config| config.scope); + if !OpenAIPreprocessor::should_apply_tool_call_format( scope, tool_choice, @@ -172,56 +269,98 @@ impl OpenAIPreprocessor { tools: &[ToolDefinition], parallel_tool_calls: Option, prompt_injected_reasoning: bool, + response_format: Option<&ResponseFormat>, preprocessed_request: &mut PreprocessedRequest, ) -> Result { let parser_name = self.tool_call_parser.as_deref(); + let explicit_config = self.runtime_config.structural_tag.as_ref(); + + let config = explicit_config.cloned().unwrap_or_default(); + let reasoning_boundary = if config.reasoning_boundary + == StructuralTagReasoningBoundary::Auto + && preprocessed_request.require_reasoning + && self.structural_tag_reasoning_metadata( + TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, + ) { + // SGLang consumes the prefix only for requests that activate its gate. + ResolvedReasoningBoundary::Backend + } else { + self.structural_tag_reasoning_boundary + }; let StructuralTagDecision::Required(builder) = structural_tag_decision( parser_name, tool_choice, tools, parallel_tool_calls, - self.runtime_config.structural_tag_mode, - self.runtime_config.structural_tag_scope, + explicit_config, + response_format.is_some_and(|format| !matches!(format, ResponseFormat::Text)), self.runtime_config.exclude_tools_when_tool_choice_none, )? else { return Ok(false); }; - // `structural_tag_decision` only returns `Required` once `parser_name` is - // confirmed `Some` (it is the source of the registry lookup that produced - // `builder`), so this is a real value, not a placeholder. let parser_name = parser_name.expect("Required decision implies a parser name"); - if matches!(tool_choice, ToolChoice::None) { - return Self::apply_tool_call_ban(builder, preprocessed_request); - } + let structured_output_schema = if matches!(tool_choice, ToolChoice::Auto) + && config.allow_tool_calls_with_structured_output + { + match response_format { + Some(ResponseFormat::JsonSchema { json_schema }) => { + Some(Cow::Borrowed(&json_schema.schema)) + } + Some(ResponseFormat::JsonObject) => { + Some(Cow::Owned(serde_json::json!({"type": "object"}))) + } + Some(ResponseFormat::Text) | None => None, + } + } else { + None + }; - // `structural_tag_decision` already confirmed `should_apply_tool_call_format` - // for this non-None tool_choice before returning `Required`. - let ctx = dynamo_parsers::tool_calling::ToolCallFormatBuildContext { + let request = StructuralTagBuildRequest { tool_choice, tools, parallel_tool_calls, - schema_mode: self.runtime_config.structural_tag_schema, - starts_in_reasoning: prompt_injected_reasoning - && !self.tool_call_structural_tag_excludes_reasoning( - preprocessed_request.require_reasoning, - ), + schema_mode: config.schema, + exclude_special_tokens: config.exclude_special_tokens, + reasoning_boundary, + tool_arguments_any_order: config.tool_arguments_any_order, + starts_in_reasoning: prompt_injected_reasoning, + structured_output_schema: structured_output_schema.as_deref(), }; - Self::apply_tool_call_format(parser_name, builder, &ctx, preprocessed_request) - } - - fn tool_call_structural_tag_excludes_reasoning(&self, require_reasoning: bool) -> bool { - // vLLM advertises exclusion for the deployment; SGLang gates it per request. - // Once the backend consumes THINK_CLOSE, this grammar must not expect it again. - // Auto tool choice alone does not activate SGLang's gate. - self.structural_tag_reasoning_metadata( - TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, - ) || (require_reasoning - && self.structural_tag_reasoning_metadata( - TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, - )) + let built_tag = match builder.build(&request) { + Err(error) + if !matches!(tool_choice, ToolChoice::None) + && structured_output_schema.is_none() => + { + tracing::warn!( + parser = parser_name, + %error, + "Failed to build structural tag; using compatibility fallback" + ); + return Ok(false); + } + result => result, + }; + let applied = apply_structural_tag(parser_name, built_tag, preprocessed_request)?; + if applied && structured_output_schema.is_some() { + // Request preprocessing materializes `response_format` as guided JSON. + // The composite structural tag now owns that schema. + preprocessed_request + .sampling_options + .guided_decoding + .get_or_insert_default() + .json = None; + } + if applied + && prompt_injected_reasoning + && config.reasoning_boundary != StructuralTagReasoningBoundary::Auto + { + preprocessed_request.require_reasoning = + reasoning_boundary == ResolvedReasoningBoundary::Backend; + } + Ok(applied) } fn structural_tag_reasoning_metadata(&self, key: &str) -> bool { @@ -239,83 +378,6 @@ impl OpenAIPreprocessor { } } - /// Find the structural tag builder for a parser, if supported. - fn structural_tag_builder_for_parser( - parser_name: &str, - ) -> Option<&'static dynamo_parsers::tool_calling::StructuralTagBuilder> { - let parser_map = dynamo_parsers::tool_calling::parsers::get_tool_parser_map(); - let builder = parser_map - .get(parser_name) - .and_then(|tc| tc.structural_tag_builder.as_ref()); - - if builder.is_none() { - tracing::debug!( - parser = parser_name, - "Structural tag enabled but parser does not support it; \ - falling back to default behaviour" - ); - } - - builder - } - - /// Apply the `tool_choice=none` ban tag, if configured. - fn apply_tool_call_ban( - builder: &dynamo_parsers::tool_calling::StructuralTagBuilder, - common_request: &mut PreprocessedRequest, - ) -> Result { - if let Some(ban_tag) = builder.build_tool_call_ban().map_err(|e| { - DynamoError::builder() - .error_type(ErrorType::Unknown) - .message(format!("failed to build tool-call ban structural tag: {e}")) - .build() - })? { - let gd = common_request - .sampling_options - .guided_decoding - .get_or_insert_default(); - gd.structural_tag = Some(ban_tag); - Ok(true) - } else { - Ok(false) - } - } - - /// Build and inject the tool-call format tag, if one is needed. - fn apply_tool_call_format( - parser_name: &str, - builder: &dynamo_parsers::tool_calling::StructuralTagBuilder, - ctx: &dynamo_parsers::tool_calling::ToolCallFormatBuildContext<'_>, - common_request: &mut PreprocessedRequest, - ) -> Result { - let structural_tag = match builder.build_tool_call_format(ctx) { - Ok(Some(tag)) => tag, - Ok(None) => { - tracing::debug!( - parser = parser_name, - "Builder returned None for structural_tag (tool_choice={:?})", - ctx.tool_choice, - ); - return Ok(false); - } - Err(e) => { - tracing::warn!( - parser = parser_name, - error = %e, - "Failed to build structural tag; using compatibility fallback" - ); - return Ok(false); - } - }; - - let gd = common_request - .sampling_options - .guided_decoding - .get_or_insert_default(); - gd.structural_tag = Some(structural_tag); - Ok(true) - } - /// Decide whether this request should use a tool-call format tag. fn should_apply_tool_call_format( scope: StructuralTagScope, @@ -337,11 +399,110 @@ impl OpenAIPreprocessor { } } +fn apply_structural_tag( + parser_name: &str, + structural_tag: anyhow::Result>, + request: &mut PreprocessedRequest, +) -> Result { + let structural_tag = match structural_tag { + Ok(Some(tag)) => tag, + Ok(None) => return Ok(false), + Err(error) => { + return Err(DynamoError::builder() + .error_type(ErrorType::Unknown) + .message(format!( + "failed to build structural_tag for parser '{parser_name}': {error}" + )) + .build()); + } + }; + + let guided_decoding = request + .sampling_options + .guided_decoding + .get_or_insert_default(); + guided_decoding.structural_tag = Some(structural_tag); + Ok(true) +} + +pub(super) fn validate_runtime_config( + runtime_config: &ModelRuntimeConfig, +) -> anyhow::Result { + let backend_excludes_reasoning = match runtime_config + .get_engine_specific::(TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY) + { + Ok(value) => value.unwrap_or(false), + Err(error) if runtime_config.structural_tag.is_none() => { + tracing::warn!( + %error, + key = TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + "Ignoring invalid structural-tag reasoning metadata" + ); + false + } + Err(error) => return Err(error), + }; + let configured_boundary = runtime_config + .structural_tag + .as_ref() + .map_or(StructuralTagReasoningBoundary::default(), |config| { + config.reasoning_boundary + }); + let reasoning_boundary = + resolve_reasoning_boundary(configured_boundary, backend_excludes_reasoning) + .map_err(anyhow::Error::msg)?; + + let Some(config) = runtime_config.structural_tag.as_ref() else { + return Ok(reasoning_boundary); + }; + + if config.reasoning_boundary == StructuralTagReasoningBoundary::Backend { + anyhow::ensure!( + runtime_config.reasoning_parser.is_some(), + "structural_tag.reasoning_boundary=backend requires a reasoning parser" + ); + } + + let v2_only_option = config + .allow_tool_calls_with_structured_output + .then_some("allow_tool_calls_with_structured_output") + .or(config + .exclude_special_tokens + .is_some() + .then_some("exclude_special_tokens")) + .or( + (config.reasoning_boundary == StructuralTagReasoningBoundary::Backend) + .then_some("reasoning_boundary"), + ) + .or(config + .tool_arguments_any_order + .then_some("tool_arguments_any_order")); + let Some(v2_only_option) = v2_only_option else { + return Ok(reasoning_boundary); + }; + + anyhow::ensure!( + v2::enabled(), + "structural_tag.{v2_only_option} requires {}=2", + env_llm::DYN_PARSER_VERSION, + ); + let parser_name = runtime_config.tool_call_parser.as_deref().ok_or_else(|| { + anyhow::anyhow!("structural_tag.{v2_only_option} requires a tool-call parser") + })?; + anyhow::ensure!( + v2::supports_family(parser_name), + "structural_tag.{v2_only_option} is not supported by parser '{parser_name}'" + ); + + Ok(reasoning_boundary) +} + #[cfg(test)] mod tests { use std::{path::PathBuf, sync::Arc}; use crate::{ + local_model::runtime_config::StructuralTagConfig, model_card::ModelDeploymentCard, protocols::common::{OutputOptions, SamplingOptions, StopConditions}, }; @@ -352,7 +513,7 @@ mod tests { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::On; + mdc.runtime_config.structural_tag = Some(StructuralTagConfig::default()); mdc.runtime_config.tool_call_parser = Some("qwen3_coder".to_string()); mdc.runtime_config.exclude_tools_when_tool_choice_none = exclude_tools_when_none; @@ -374,7 +535,7 @@ mod tests { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::On; + mdc.runtime_config.structural_tag = Some(StructuralTagConfig::default()); mdc.runtime_config.tool_call_parser = Some("kimi_k2".to_string()); if let Some(excludes_reasoning) = excludes_reasoning { mdc.runtime_config @@ -392,7 +553,6 @@ mod tests { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; mdc.runtime_config.tool_call_parser = Some("kimi_k3".to_string()); OpenAIPreprocessor::new(mdc).unwrap() } @@ -413,6 +573,7 @@ mod tests { &tools, None, true, + None, &mut request, ) .unwrap() @@ -446,154 +607,273 @@ mod tests { } #[test] - fn kimi_k3_reasoning_gate_avoids_duplicate_prefix_per_request() { + fn reasoning_boundary_resolution_follows_backend_policy_and_rejects_conflicts() { + assert_eq!( + resolve_reasoning_boundary(StructuralTagReasoningBoundary::Auto, false).unwrap(), + ResolvedReasoningBoundary::StructuralTag + ); + assert_eq!( + resolve_reasoning_boundary(StructuralTagReasoningBoundary::Auto, true).unwrap(), + ResolvedReasoningBoundary::Backend + ); + assert_eq!( + resolve_reasoning_boundary(StructuralTagReasoningBoundary::Backend, false).unwrap(), + ResolvedReasoningBoundary::Backend + ); + assert!( + resolve_reasoning_boundary(StructuralTagReasoningBoundary::StructuralTag, true) + .is_err() + ); + } + + #[test] + fn reasoning_boundary_is_resolved_when_the_preprocessor_is_created() { + assert_eq!( + kimi_k2_preprocessor(None).structural_tag_reasoning_boundary, + ResolvedReasoningBoundary::StructuralTag + ); + assert_eq!( + kimi_k2_preprocessor(Some(true)).structural_tag_reasoning_boundary, + ResolvedReasoningBoundary::Backend + ); + } + + #[test] + fn conflicting_reasoning_boundary_is_rejected_when_the_preprocessor_is_created() { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); - let tools = [ToolDefinition { - name: "get_weather".to_string(), - parameters: Some(serde_json::json!({ - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"] - })), - strict: None, - }]; - for (gate, require_reasoning, prompt_injected_reasoning, choice, expects_prefix) in [ - (Some(true), true, true, ToolChoice::Required, false), - (None, true, true, ToolChoice::Required, true), - (Some(false), true, true, ToolChoice::Required, true), - (Some(true), false, true, ToolChoice::Auto, true), - (Some(true), true, false, ToolChoice::Required, false), - ( - Some(true), - false, - false, - ToolChoice::Named("get_weather".to_string()), - false, - ), - ] { - let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::On; - mdc.runtime_config.structural_tag_scope = StructuralTagScope::Always; - mdc.runtime_config.tool_call_parser = Some("kimi_k3".to_string()); - if let Some(gate) = gate { - mdc.runtime_config - .set_engine_specific(TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, gate) - .unwrap(); - } - let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); - let mut request = preprocessed_request(); - request.require_reasoning = require_reasoning; - assert!( - preprocessor - .apply_tool_choice_structural_tag( - &choice, - &tools, - None, - prompt_injected_reasoning, - &mut request, - ) - .unwrap(), - "gate={gate:?}, require_reasoning={require_reasoning}, choice={choice:?}" - ); - let tag = request - .sampling_options - .guided_decoding - .unwrap() - .structural_tag - .unwrap(); - let has_prefix = tag["format"]["elements"][0]["end"] == "<|close|>think<|sep|>"; - assert_eq!( - has_prefix, expects_prefix, - "gate={gate:?}, require_reasoning={require_reasoning}, prompt_injected_reasoning={prompt_injected_reasoning}, choice={choice:?}" - ); - assert!(tag.to_string().contains("<|open|>tools<|sep|>")); - } + let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); + mdc.runtime_config.structural_tag = Some(StructuralTagConfig { + reasoning_boundary: StructuralTagReasoningBoundary::StructuralTag, + ..Default::default() + }); + mdc.runtime_config + .set_engine_specific( + TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + true, + ) + .unwrap(); + + let error = OpenAIPreprocessor::new(mdc) + .err() + .expect("the conflicting boundary must fail during preprocessor initialization"); + assert!(error.to_string().contains( + "structural_tag.reasoning_boundary=structural_tag conflicts with the backend" + )); } #[test] - fn kimi_forced_choices_are_classified_as_native_format_fallbacks() { + fn named_kimi_k3_is_intrinsic_even_when_global_mode_is_off() { let named = ToolChoice::Named("get_weather".to_string()); - assert!(requires_native_tool_call_format( + assert!(requires_intrinsic_structural_tag(Some("kimi_k3"), &named)); + assert!(requires_intrinsic_structural_tag(Some("kimi-k3"), &named)); + } + + #[test] + fn forced_kimi_k2_is_intrinsic_even_when_global_mode_is_off() { + assert!(requires_intrinsic_structural_tag( Some("kimi_k2"), &ToolChoice::Required )); - assert!(requires_native_tool_call_format(Some("kimi_k2"), &named)); - assert!(requires_native_tool_call_format( - Some("kimi_k3"), - &ToolChoice::Required + assert!(requires_intrinsic_structural_tag( + Some("kimi_k2"), + &ToolChoice::Named("get_weather".to_string()) )); - assert!(requires_native_tool_call_format(Some("kimi_k3"), &named)); - assert!(requires_native_tool_call_format(Some("kimi-k3"), &named)); - assert!(!requires_native_tool_call_format( + } + + // Durable registry-parity property: every parser/tool_choice combination the + // intrinsic predicate recognizes must have a real, registered + // `StructuralTagBuilder` in the parser registry. If a future edit registers a + // new intrinsic-family alias, or de-registers an existing one, without keeping + // both in lockstep, `structural_tag_decision` would otherwise report `Required` + // with no way to actually build the tag — this is the exact gap the shared + // decision owner exists to close. No global-state mutation, no scratch worktree, + // runs against the real production registry. + #[test] + fn every_intrinsic_parser_choice_has_a_registered_structural_tag_builder() { + let cases: &[(&str, ToolChoice)] = &[ + ("kimi_k2", ToolChoice::Required), + ("kimi_k2", ToolChoice::Named("get_weather".to_string())), + ("kimi_k3", ToolChoice::Named("get_weather".to_string())), + ("kimi-k3", ToolChoice::Named("get_weather".to_string())), + ]; + for (parser, choice) in cases { + assert!( + requires_intrinsic_structural_tag(Some(parser), choice), + "test fixture drifted: '{parser}' + {choice:?} is no longer intrinsic" + ); + assert!( + SelectedStructuralTagBuilder::for_parser(parser).is_some(), + "'{parser}' is intrinsic per the predicate but the parser registry has \ + no structural-tag builder for it — the predicate and the registry have \ + drifted apart" + ); + } + } + + #[test] + fn other_choices_and_parsers_still_follow_the_global_mode() { + assert!(!requires_intrinsic_structural_tag( Some("kimi_k3"), - &ToolChoice::Auto + &ToolChoice::Required )); - assert!(!requires_native_tool_call_format( + assert!(!requires_intrinsic_structural_tag( Some("hermes"), &ToolChoice::Named("get_weather".to_string()) )); - assert!(!requires_native_tool_call_format( + assert!(!requires_intrinsic_structural_tag( Some("kimi_k2"), &ToolChoice::Auto )); } #[test] - fn kimi_forced_choices_keep_native_tags_when_global_mode_is_off() { - let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + fn tools_with_structured_output_require_a_tag_independently_of_scope() { let tools = [ToolDefinition { name: "get_weather".to_string(), - parameters: Some(serde_json::json!({ - "type": "object", - "properties": {"location": {"type": "string"}}, - "required": ["location"] - })), + parameters: None, strict: None, }]; - for (parser, choice, marker) in [ - ( - "kimi_k2", - ToolChoice::Required, - "<|tool_calls_section_begin|>", - ), - ( - "kimi_k2", - ToolChoice::Named("get_weather".to_string()), - "<|tool_calls_section_begin|>", - ), - ( - "kimi_k3", - ToolChoice::Named("get_weather".to_string()), - "<|open|>call", - ), - ( - "kimi-k3", - ToolChoice::Named("get_weather".to_string()), - "<|open|>call", - ), - ] { - let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; - mdc.runtime_config.tool_call_parser = Some(parser.to_string()); - let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); - let mut request = preprocessed_request(); + let config = StructuralTagConfig { + scope: StructuralTagScope::Auto, + allow_tool_calls_with_structured_output: true, + ..Default::default() + }; - let applied = preprocessor - .apply_tool_choice_structural_tag(&choice, &tools, None, false, &mut request) - .unwrap(); + let decision = structural_tag_decision( + Some("qwen3_coder"), + &ToolChoice::Auto, + &tools, + None, + Some(&config), + true, + true, + ) + .unwrap(); - assert!(applied, "{parser} + {choice:?} must retain its native tag"); - let tag = request - .sampling_options - .guided_decoding - .as_ref() - .and_then(|guided| guided.structural_tag.as_ref()) - .expect("forced Kimi choice must install a native structural tag"); - assert!(tag.to_string().contains(marker)); - assert!(tag.to_string().contains("get_weather")); + assert!(decision.is_required()); + } + + #[test] + fn structured_output_requires_explicit_tool_call_composition_opt_in() { + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: None, + strict: None, + }]; + + for scope in [StructuralTagScope::Auto, StructuralTagScope::Always] { + let config = StructuralTagConfig { + scope, + allow_tool_calls_with_structured_output: false, + ..Default::default() + }; + let decision = structural_tag_decision( + Some("qwen3_coder"), + &ToolChoice::Auto, + &tools, + None, + Some(&config), + true, + true, + ) + .unwrap(); + + assert!(!decision.is_required()); } + + let config = StructuralTagConfig { + scope: StructuralTagScope::Auto, + allow_tool_calls_with_structured_output: true, + ..Default::default() + }; + let decision = structural_tag_decision( + Some("qwen3_coder"), + &ToolChoice::Auto, + &tools, + None, + Some(&config), + false, + true, + ) + .unwrap(); + + assert!(!decision.is_required()); + } + + #[test] + fn structured_output_takes_precedence_over_a_tool_call_ban() { + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: None, + strict: None, + }]; + let config = StructuralTagConfig::default(); + + let decision = structural_tag_decision( + Some("qwen3_coder"), + &ToolChoice::None, + &tools, + None, + Some(&config), + true, + false, + ) + .unwrap(); + + assert!(!decision.is_required()); + } + + #[test] + fn kimi_k2_required_installs_native_tag_when_global_mode_is_off() { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); + mdc.runtime_config.tool_call_parser = Some("kimi_k2".to_string()); + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: Some(serde_json::json!({ + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"] + })), + strict: None, + }]; + let mut request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_structural_tag( + &ToolChoice::Required, + &tools, + None, + false, + None, + &mut request, + ) + .unwrap(); + + assert!(applied); + let format = &request + .sampling_options + .guided_decoding + .as_ref() + .unwrap() + .structural_tag + .as_ref() + .unwrap()["format"]; + assert_eq!(format["type"], "sequence"); + assert_eq!( + format["elements"][0]["value"], + "<|tool_calls_section_begin|>" + ); + assert_eq!(format["elements"][1]["type"], "tags_with_separator"); + assert_eq!(format["elements"][1]["at_least_one"], true); + assert_eq!( + format["elements"][1]["tags"][0]["begin"], + "<|tool_call_begin|>functions.get_weather:" + ); + assert_eq!(format["elements"][2]["value"], "<|tool_calls_section_end|>"); } #[test] @@ -612,6 +892,7 @@ mod tests { &tools, None, false, + None, &mut request, ) .unwrap(); @@ -634,7 +915,14 @@ mod tests { let mut request = preprocessed_request(); let err = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::Required, &[], None, false, &mut request) + .apply_tool_choice_structural_tag( + &ToolChoice::Required, + &[], + None, + false, + None, + &mut request, + ) .expect_err( "kimi_k2 + required with no tools must be rejected, not silently \ resolved to a structural tag", @@ -649,7 +937,14 @@ mod tests { let mut request = preprocessed_request(); let err = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::Required, &[], None, false, &mut request) + .apply_tool_choice_structural_tag( + &ToolChoice::Required, + &[], + None, + false, + None, + &mut request, + ) .expect_err("required with no tools must be rejected before Kimi K3's XTML path"); assert_eq!(err.error_type(), ErrorType::InvalidArgument); assert!(request.sampling_options.guided_decoding.is_none()); @@ -671,6 +966,7 @@ mod tests { &tools, None, false, + None, &mut request, ) .expect_err( @@ -687,9 +983,16 @@ mod tests { let mut request = preprocessed_request(); let err = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::Required, &[], None, false, &mut request) + .apply_tool_choice_structural_tag( + &ToolChoice::Required, + &[], + None, + false, + None, + &mut request, + ) .expect_err( - "operator-enabled structural_tag_mode must not let required-with-no-tools \ + "operator-configured structural tags must not let required-with-no-tools \ through for a non-Kimi registry-supported parser either", ); assert_eq!(err.error_type(), ErrorType::InvalidArgument); @@ -712,6 +1015,7 @@ mod tests { &tools, None, false, + None, &mut request, ) .expect_err( @@ -730,24 +1034,274 @@ mod tests { let mut request = preprocessed_request(); let applied = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::None, &[], None, false, &mut request) + .apply_tool_choice_structural_tag( + &ToolChoice::None, + &[], + None, + false, + None, + &mut request, + ) .unwrap(); assert!(!applied); let mut request = preprocessed_request(); let applied = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::Auto, &[], None, false, &mut request) + .apply_tool_choice_structural_tag( + &ToolChoice::Auto, + &[], + None, + false, + None, + &mut request, + ) .unwrap(); assert!(!applied); } + #[test] + fn tool_choice_none_skips_ban_only_when_prompt_excludes_tools() { + const CHILD: &str = "DYNAMO_STRUCTURAL_TAG_NONE_TEST_CHILD"; + if std::env::var_os(CHILD).is_none() { + for enabled in ["1", "2"] { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "preprocessor::structural_tag::tests::tool_choice_none_skips_ban_only_when_prompt_excludes_tools", + "--nocapture", + ]) + .env(CHILD, enabled) + .env(env_llm::DYN_PARSER_VERSION, enabled) + .output() + .unwrap(); + assert!( + output.status.success() + && String::from_utf8_lossy(&output.stdout).contains("running 1 test"), + "parsers v2={enabled}:\n{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + } + return; + } + assert_eq!(v2::enabled(), std::env::var(CHILD).unwrap() == "2"); + + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: None, + strict: None, + }]; + + let preprocessor = structural_tag_preprocessor(true); + let mut request = preprocessed_request(); + let applied = preprocessor + .apply_tool_choice_structural_tag( + &ToolChoice::None, + &tools, + None, + false, + None, + &mut request, + ) + .unwrap(); + + assert!(!applied); + assert!(request.sampling_options.guided_decoding.is_none()); + + let preprocessor = structural_tag_preprocessor(false); + let mut request = preprocessed_request(); + let applied = preprocessor + .apply_tool_choice_structural_tag( + &ToolChoice::None, + &tools, + None, + false, + None, + &mut request, + ) + .unwrap(); + + assert!(applied); + let structural_tag = request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.structural_tag.as_ref()) + .expect("tool-call ban should be installed"); + let expected_content = if v2::enabled() { + serde_json::json!({ + "type": "any_text", + "excludes": ["", ""] + }) + }; + assert_eq!(structural_tag["format"]["content"], expected_content); + } + + #[test] + fn kimi_k3_reasoning_gate_avoids_duplicate_prefix_per_request() { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: Some(serde_json::json!({ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + })), + strict: None, + }]; + for (gate, require_reasoning, prompt_injected_reasoning, choice, expects_prefix) in [ + (Some(true), true, true, ToolChoice::Required, false), + (None, true, true, ToolChoice::Required, true), + (Some(false), true, true, ToolChoice::Required, true), + (Some(true), false, true, ToolChoice::Auto, true), + (Some(true), true, false, ToolChoice::Required, false), + ( + Some(true), + false, + false, + ToolChoice::Named("get_weather".to_string()), + false, + ), + ] { + let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); + mdc.runtime_config.structural_tag = Some(StructuralTagConfig { + scope: StructuralTagScope::Always, + ..Default::default() + }); + mdc.runtime_config.tool_call_parser = Some("kimi_k3".to_string()); + if let Some(gate) = gate { + mdc.runtime_config + .set_engine_specific(TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, gate) + .unwrap(); + } + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let mut request = preprocessed_request(); + request.require_reasoning = require_reasoning; + assert!( + preprocessor + .apply_tool_choice_structural_tag( + &choice, + &tools, + None, + prompt_injected_reasoning, + None, + &mut request, + ) + .unwrap(), + "gate={gate:?}, require_reasoning={require_reasoning}, choice={choice:?}" + ); + let tag = request + .sampling_options + .guided_decoding + .unwrap() + .structural_tag + .unwrap(); + let has_prefix = tag["format"]["elements"][0]["end"] == "<|close|>think<|sep|>"; + assert_eq!( + has_prefix, expects_prefix, + "gate={gate:?}, require_reasoning={require_reasoning}, prompt_injected_reasoning={prompt_injected_reasoning}, choice={choice:?}" + ); + assert!(tag.to_string().contains("<|open|>tools<|sep|>")); + } + } + + #[test] + fn kimi_forced_choices_are_classified_as_native_format_fallbacks() { + let named = ToolChoice::Named("get_weather".to_string()); + assert!(requires_native_tool_call_format( + Some("kimi_k2"), + &ToolChoice::Required + )); + assert!(requires_native_tool_call_format(Some("kimi_k2"), &named)); + assert!(requires_native_tool_call_format( + Some("kimi_k3"), + &ToolChoice::Required + )); + assert!(requires_native_tool_call_format(Some("kimi_k3"), &named)); + assert!(requires_native_tool_call_format(Some("kimi-k3"), &named)); + assert!(!requires_native_tool_call_format( + Some("kimi_k3"), + &ToolChoice::Auto + )); + assert!(!requires_native_tool_call_format( + Some("hermes"), + &ToolChoice::Named("get_weather".to_string()) + )); + assert!(!requires_native_tool_call_format( + Some("kimi_k2"), + &ToolChoice::Auto + )); + } + + #[test] + fn kimi_forced_choices_keep_native_tags_when_global_mode_is_off() { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: Some(serde_json::json!({ + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"] + })), + strict: None, + }]; + for (parser, choice, marker) in [ + ( + "kimi_k2", + ToolChoice::Required, + "<|tool_calls_section_begin|>", + ), + ( + "kimi_k2", + ToolChoice::Named("get_weather".to_string()), + "<|tool_calls_section_begin|>", + ), + ( + "kimi_k3", + ToolChoice::Named("get_weather".to_string()), + "<|open|>call", + ), + ( + "kimi-k3", + ToolChoice::Named("get_weather".to_string()), + "<|open|>call", + ), + ] { + let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); + mdc.runtime_config.structural_tag = None; + mdc.runtime_config.tool_call_parser = Some(parser.to_string()); + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let mut request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_structural_tag(&choice, &tools, None, false, None, &mut request) + .unwrap(); + + assert!(applied, "{parser} + {choice:?} must retain its native tag"); + let tag = request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.structural_tag.as_ref()) + .expect("forced Kimi choice must install a native structural tag"); + assert!(tag.to_string().contains(marker)); + assert!(tag.to_string().contains("get_weather")); + } + } + #[test] fn global_mode_off_is_authoritative_for_kimi_k3_auto() { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; - mdc.runtime_config.structural_tag_scope = StructuralTagScope::Auto; + mdc.runtime_config.structural_tag = None; mdc.runtime_config.tool_call_parser = Some("kimi_k3".to_string()); let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); let tools = [ToolDefinition { @@ -764,7 +1318,14 @@ mod tests { let mut request = preprocessed_request(); let applied = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::Auto, &tools, None, false, &mut request) + .apply_tool_choice_structural_tag( + &ToolChoice::Auto, + &tools, + None, + false, + None, + &mut request, + ) .unwrap(); assert!(!applied); @@ -792,40 +1353,4 @@ mod tests { None, )); } - - #[test] - fn tool_choice_none_skips_ban_only_when_prompt_excludes_tools() { - let tools = [ToolDefinition { - name: "get_weather".to_string(), - parameters: None, - strict: None, - }]; - - let preprocessor = structural_tag_preprocessor(true); - let mut request = preprocessed_request(); - let applied = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::None, &tools, None, false, &mut request) - .unwrap(); - - assert!(!applied); - assert!(request.sampling_options.guided_decoding.is_none()); - - let preprocessor = structural_tag_preprocessor(false); - let mut request = preprocessed_request(); - let applied = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::None, &tools, None, false, &mut request) - .unwrap(); - - assert!(applied); - let structural_tag = request - .sampling_options - .guided_decoding - .as_ref() - .and_then(|guided| guided.structural_tag.as_ref()) - .expect("tool-call ban should be installed"); - assert_eq!( - structural_tag["format"]["content"]["exclude_tokens"], - serde_json::json!([""]) - ); - } } diff --git a/lib/llm/src/preprocessor/structural_tag/v1.rs b/lib/llm/src/preprocessor/structural_tag/v1.rs new file mode 100644 index 000000000000..6949f295b75a --- /dev/null +++ b/lib/llm/src/preprocessor/structural_tag/v1.rs @@ -0,0 +1,46 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use dynamo_parsers::tool_calling::{ + StructuralTagBuilder as InnerBuilder, ToolCallFormatBuildContext, +}; + +use super::{ResolvedReasoningBoundary, StructuralTagBuildRequest}; + +#[derive(Clone, Copy)] +pub(crate) struct StructuralTagBuilder(&'static InnerBuilder); + +impl StructuralTagBuilder { + pub(super) fn for_parser(parser_name: &str) -> Option { + dynamo_parsers::tool_calling::parsers::get_tool_parser_map() + .get(parser_name) + .and_then(|config| config.structural_tag_builder.as_ref()) + .map(Self) + } + + pub(super) fn build( + self, + request: &StructuralTagBuildRequest<'_>, + ) -> anyhow::Result> { + anyhow::ensure!( + request.structured_output_schema.is_none(), + "tool calls with structured output require parsers v2" + ); + + if matches!( + request.tool_choice, + dynamo_parsers::tool_calling::ToolChoice::None + ) { + return self.0.build_tool_call_ban(); + } + + self.0.build_tool_call_format(&ToolCallFormatBuildContext { + tool_choice: request.tool_choice, + tools: request.tools, + parallel_tool_calls: request.parallel_tool_calls, + schema_mode: request.schema_mode, + starts_in_reasoning: request.starts_in_reasoning + && request.reasoning_boundary == ResolvedReasoningBoundary::StructuralTag, + }) + } +} diff --git a/lib/llm/src/preprocessor/structural_tag/v2.rs b/lib/llm/src/preprocessor/structural_tag/v2.rs new file mode 100644 index 000000000000..fc54150f5d7c --- /dev/null +++ b/lib/llm/src/preprocessor/structural_tag/v2.rs @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use dynamo_parsers::tool_calling::{ToolChoice, ToolDefinition}; +use dynamo_parsers_v2::Tool; +use dynamo_parsers_v2::structural_tag::{ + ReasoningBoundary as ReasoningBoundaryV2, StructuralTagBuilder as InnerBuilder, + StructuralTagContext, StructuralTagOptions, + StructuralTagSchemaMode as StructuralTagSchemaModeV2, + StructuralTagToolChoice as StructuralTagToolChoiceV2, +}; + +use crate::protocols::openai::chat_completions::tool_parser_v2; + +use super::StructuralTagBuildRequest; + +#[derive(Clone, Copy)] +pub(crate) struct StructuralTagBuilder(&'static InnerBuilder); + +impl StructuralTagBuilder { + pub(super) fn for_parser(parser_name: &str) -> Option { + dynamo_parsers_v2::structural_tag_builder_for_family(tool_parser_v2::canonical_family( + parser_name, + )) + .map(Self) + } + + pub(super) fn build( + self, + request: &StructuralTagBuildRequest<'_>, + ) -> anyhow::Result> { + let tools = request.tools.iter().map(to_v2_tool).collect::>(); + let tool_choice = match request.tool_choice { + ToolChoice::None => StructuralTagToolChoiceV2::None, + ToolChoice::Auto => StructuralTagToolChoiceV2::Auto, + ToolChoice::Required => StructuralTagToolChoiceV2::Required, + ToolChoice::Named(name) => StructuralTagToolChoiceV2::Named(name), + }; + let schema_mode = match request.schema_mode { + dynamo_parsers::tool_calling::StructuralTagSchemaMode::Auto => { + StructuralTagSchemaModeV2::Auto + } + dynamo_parsers::tool_calling::StructuralTagSchemaMode::Strict => { + StructuralTagSchemaModeV2::Strict + } + }; + let reasoning_boundary = match request.reasoning_boundary { + super::ResolvedReasoningBoundary::StructuralTag => ReasoningBoundaryV2::StructuralTag, + super::ResolvedReasoningBoundary::Backend => ReasoningBoundaryV2::External, + }; + + self.0.build_with_options( + &StructuralTagContext { + tool_choice, + tools: &tools, + parallel_tool_calls: request.parallel_tool_calls, + schema_mode, + structured_output_schema: request.structured_output_schema, + starts_in_reasoning: request.starts_in_reasoning, + }, + &StructuralTagOptions { + exclude_special_tokens: request.exclude_special_tokens, + reasoning_boundary, + tool_arguments_any_order: request.tool_arguments_any_order, + }, + ) + } +} + +fn to_v2_tool(tool: &ToolDefinition) -> Tool { + Tool { + name: tool.name.clone(), + description: None, + parameters: tool.parameters.clone().unwrap_or(serde_json::Value::Null), + // Match the deployment policy shared with v1: only explicit false opts out. + strict: Some(tool.strict.unwrap_or(true)), + } +} + +pub(super) fn enabled() -> bool { + tool_parser_v2::enabled() +} + +pub(super) fn supports_family(parser_name: &str) -> bool { + StructuralTagBuilder::for_parser(parser_name).is_some() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn omitted_tool_strict_matches_v1_schema_enforcement() { + for (strict, expected) in [(None, true), (Some(true), true), (Some(false), false)] { + let tool = ToolDefinition { + name: "get_weather".to_string(), + parameters: None, + strict, + }; + assert_eq!(to_v2_tool(&tool).strict, Some(expected)); + } + } + + #[test] + fn deepseek_v4_engine_aliases_select_the_structural_tag_builder() { + for alias in ["deepseek_v4", "deepseek-v4", "deepseekv4"] { + assert!(supports_family(alias), "unsupported alias: {alias}"); + } + } +} diff --git a/lib/llm/src/preprocessor/tool_choice.rs b/lib/llm/src/preprocessor/tool_choice.rs index 403d488dac73..aee66437b1c5 100644 --- a/lib/llm/src/preprocessor/tool_choice.rs +++ b/lib/llm/src/preprocessor/tool_choice.rs @@ -19,12 +19,14 @@ use dynamo_protocols::types::{ }; use dynamo_runtime::error::{DynamoError, ErrorType}; -/// Tool names and parser diagnostics can contain request data, so this helper does not mark its message public. -fn invalid_argument(message: impl Into) -> DynamoError { - DynamoError::builder() +fn invalid_argument(message: impl Into, public_message: Option<&str>) -> DynamoError { + let mut error = DynamoError::builder() .error_type(ErrorType::InvalidArgument) - .message(message) - .build() + .message(message); + if let Some(public_message) = public_message { + error = error.public_message(public_message); + } + error.build() } impl OpenAIPreprocessor { @@ -83,18 +85,21 @@ impl OpenAIPreprocessor { let has_response_format_constraint = has_response_format_constraint(request); let parser_tool_choice = convert_tool_choice(tool_choice); + self.validate_structured_output_composition(request, tool_choice, tools.as_ref())?; + if is_forced_tool_choice && has_explicit_guided_decoding { - return Err(invalid_argument(concat!( - "guided decoding cannot be used in the same request as ", - "tool_choice=\"required\" or a named tool_choice.", - ))); + return Err(invalid_argument( + concat!( + "guided decoding cannot be used in the same request as ", + "tool_choice=\"required\" or a named tool_choice.", + ), + None, + )); } - // For non-forced tool choice, explicit guided decoding and response_format - // constrain assistant content, so tool-choice guided decoding stays inactive. - let has_assistant_constraint = - has_explicit_guided_decoding || has_response_format_constraint; - if !is_forced_tool_choice && has_assistant_constraint { + // Explicit guided decoding constrains assistant content and cannot be + // composed with structural-tag tool calls. + if !is_forced_tool_choice && has_explicit_guided_decoding { return Ok(GuidedToolConstraint::None); } @@ -111,11 +116,18 @@ impl OpenAIPreprocessor { &convert_tools(tools.as_ref()), request.inner.parallel_tool_calls, prompt_injected_reasoning, + request.inner.response_format.as_ref(), common_request, )? { return Ok(GuidedToolConstraint::StructuralTag); } + // If the structural-tag layer did not compose this response format with + // auto tool calls, it remains the sole assistant-output constraint. + if !is_forced_tool_choice && has_response_format_constraint { + return Ok(GuidedToolConstraint::None); + } + let requires_native_format = requires_native_tool_call_format(self.tool_call_parser.as_deref(), &parser_tool_choice) || requires_native_tool_call_format( @@ -152,7 +164,7 @@ impl OpenAIPreprocessor { } Ok(None) => {} Err(err) => { - return Err(invalid_argument(err.to_string())); + return Err(invalid_argument(err.to_string(), None)); } } @@ -160,6 +172,35 @@ impl OpenAIPreprocessor { // tool-choice JSON fallback were needed. Ok(GuidedToolConstraint::None) } + + fn validate_structured_output_composition( + &self, + request: &NvCreateChatCompletionRequest, + tool_choice: &ChatCompletionToolChoiceOption, + tools: &[ChatCompletionTool], + ) -> Result<(), DynamoError> { + let allow_tool_calls_with_structured_output = self + .runtime_config + .structural_tag + .as_ref() + .is_some_and(|config| config.allow_tool_calls_with_structured_output); + + if matches!(tool_choice, ChatCompletionToolChoiceOption::Auto) + && !tools.is_empty() + && allow_tool_calls_with_structured_output + { + validate::validate_response_format_conflicts( + &request.inner.response_format, + &request.common, + ) + .map_err(|error| { + let message = error.to_string(); + invalid_argument(&message, Some(&message)) + })?; + } + + Ok(()) + } } fn has_explicit_guided_decoding(request: &NvCreateChatCompletionRequest) -> bool { @@ -214,7 +255,7 @@ pub(crate) fn effective_tools( request: &CreateChatCompletionRequest, ) -> Result, DynamoError> { validate::validated_effective_tools(request) - .map_err(|error| invalid_argument(error.to_string())) + .map_err(|error| invalid_argument(error.to_string(), None)) } pub(crate) fn effective_tool_definitions( @@ -268,7 +309,7 @@ pub(crate) fn guided_tool_constraint_with_effective_tools( .as_ref() .unwrap_or(&ChatCompletionToolChoiceOption::Auto); validate_openai_tool_choice(Some(tool_choice), Some(tools)) - .map_err(|error| invalid_argument(error.to_string()))?; + .map_err(|error| invalid_argument(error.to_string(), None))?; let is_forced_tool_choice = matches!( tool_choice, ChatCompletionToolChoiceOption::Required | ChatCompletionToolChoiceOption::Named(_) @@ -303,7 +344,7 @@ pub(crate) fn guided_tool_constraint_with_effective_tools( ) { Ok(Some(_)) => Ok(installed_json_constraint(tool_choice)), Ok(None) => Ok(GuidedToolConstraint::None), - Err(e) => Err(invalid_argument(e.to_string())), + Err(e) => Err(invalid_argument(e.to_string(), None)), } } @@ -365,7 +406,8 @@ mod tests { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = mode; + mdc.runtime_config.structural_tag = (mode == StructuralTagMode::On) + .then(crate::local_model::runtime_config::StructuralTagConfig::default); mdc.runtime_config.tool_call_parser = Some(parser.to_string()); OpenAIPreprocessor::new(mdc).unwrap() } diff --git a/lib/llm/src/protocols/openai.rs b/lib/llm/src/protocols/openai.rs index ba1ee7eef729..0b677fc791c1 100644 --- a/lib/llm/src/protocols/openai.rs +++ b/lib/llm/src/protocols/openai.rs @@ -452,16 +452,11 @@ pub struct ParsingOptions { #[serde(default)] pub move_reasoning_to_content_when_empty: bool, - /// The worker's operator-configured structural-tag policy. Carried through so - /// the HTTP-layer tool-call-gate reconstruction - /// (`http::service::apply_request_tool_call_parsing_options`) can consult the - /// same structural-tag contract the real preprocessing path uses, instead of - /// only recognizing intrinsically-forced model families. - #[serde(default)] - pub structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode, - - #[serde(default)] - pub structural_tag_scope: crate::local_model::runtime_config::StructuralTagScope, + /// The worker's structural-tag policy. Carried through so the HTTP-layer + /// tool-call-gate reconstruction consults the same configuration as the + /// request preprocessor. Presence enables operator-controlled tags. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub structural_tag: Option, #[serde( default = "crate::local_model::runtime_config::default_exclude_tools_when_tool_choice_none" @@ -499,8 +494,7 @@ impl ParsingOptions { guided_tool_constraint: GuidedToolConstraint::None, parallel_tool_calls: None, move_reasoning_to_content_when_empty: false, - structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::default(), - structural_tag_scope: crate::local_model::runtime_config::StructuralTagScope::default(), + structural_tag: None, exclude_tools_when_tool_choice_none: crate::local_model::runtime_config::default_exclude_tools_when_tool_choice_none(), tools: Vec::new(), diff --git a/lib/llm/src/protocols/openai/chat_completions/tool_parser_v2.rs b/lib/llm/src/protocols/openai/chat_completions/tool_parser_v2.rs index 3b1d50a7bccb..c2aba138fa42 100644 --- a/lib/llm/src/protocols/openai/chat_completions/tool_parser_v2.rs +++ b/lib/llm/src/protocols/openai/chat_completions/tool_parser_v2.rs @@ -71,6 +71,16 @@ fn parser_version_value(value: Option<&str>) -> anyhow::Result { ParserVersion::try_from(value) } +/// Map engine-facing parser aliases onto the family keys used by +/// `dynamo-parsers-v2`. +pub(crate) fn canonical_family(family: &str) -> &str { + match family { + "deepseek-v4" | "deepseekv4" => "deepseek_v4", + family => family, + } +} + +/// Whether the deployment explicitly selects parsers v2. pub(crate) fn enabled() -> bool { match selected_version() { Ok(ParserVersion::V2) => true, @@ -171,7 +181,7 @@ pub(crate) fn parse_complete( family: &str, ) -> anyhow::Result<(Vec, String)> { let v2_tools = to_v2_tools(tools); - let mut parser = create_tool_parser_for_family(family, &v2_tools)?; + let mut parser = create_tool_parser_for_family(canonical_family(family), &v2_tools)?; let result = parser.parse_complete(content)?; let tool_calls = result @@ -472,6 +482,7 @@ pub(crate) fn apply_stream( where S: Stream> + Send + 'static, { + let family = canonical_family(&family).to_string(); let v2_tools = to_v2_tools(tool_definitions.as_deref()); stream! { // The caller only routes supported families here, but if a parser cannot be @@ -825,6 +836,14 @@ mod tests { const MUSE_TOOL: &str = "<|start|>assistant to=get_weather<|message|>Paris<|eom|>"; const MUSE_ANSWER: &str = "<|start|>assistant to=user<|message|>It's 18C.<|eot|>"; + #[test] + fn deepseek_v4_engine_aliases_select_the_v2_family() { + for alias in ["deepseek_v4", "deepseek-v4", "deepseekv4"] { + assert!(supports_family(alias), "unsupported alias: {alias}"); + assert_eq!(canonical_family(alias), "deepseek_v4"); + } + } + fn muse_turn() -> String { format!("{MUSE_REASONING}{MUSE_TOOL}{MUSE_ANSWER}") } diff --git a/lib/llm/src/protocols/openai/validate.rs b/lib/llm/src/protocols/openai/validate.rs index d17940ddf52e..8188e89bc997 100644 --- a/lib/llm/src/protocols/openai/validate.rs +++ b/lib/llm/src/protocols/openai/validate.rs @@ -11,8 +11,10 @@ use dynamo_runtime::config::{ }; use serde_json::Value; -use super::common_ext::{CommonExtProvider, extract_guided_decoding_options}; -use super::tools::{ToolChoiceError, validate_openai_tool_choice}; +use super::{ + common_ext::{CommonExt, CommonExtProvider, extract_guided_decoding_options}, + tools::{ToolChoiceError, validate_openai_tool_choice}, +}; // // Hyperparameter Contraints @@ -251,6 +253,34 @@ pub fn validate_response_format( } } +/// Reject multiple independent constraints on assistant output. +pub fn validate_response_format_conflicts( + response_format: &Option, + common: &CommonExt, +) -> Result<(), anyhow::Error> { + use dynamo_protocols::types::ResponseFormat; + + let has_response_format_constraint = response_format + .as_ref() + .is_some_and(|format| !matches!(format, ResponseFormat::Text)); + let has_explicit_guided_decoding = common.guided_json.is_some() + || common.guided_regex.is_some() + || common + .guided_choice + .as_ref() + .is_some_and(|choices| !choices.is_empty()) + || common.guided_grammar.is_some(); + + if has_response_format_constraint && has_explicit_guided_decoding { + anyhow::bail!( + "`response_format` cannot be used together with `guided_json`, `guided_regex`, \ + `guided_choice`, or `guided_grammar`" + ); + } + + Ok(()) +} + /// Validates the temperature parameter pub fn validate_temperature(temperature: Option) -> Result<(), anyhow::Error> { if let Some(temp) = temperature @@ -1056,6 +1086,61 @@ mod tests { HashMap::from([("experimental_field".to_string(), json!("value"))]) } + fn explicit_guided_decoding_options() -> Vec { + vec![ + CommonExt { + guided_json: Some(json!({"type": "object"})), + ..Default::default() + }, + CommonExt { + guided_regex: Some("[a-z]+".to_string()), + ..Default::default() + }, + CommonExt { + guided_choice: Some(vec!["yes".to_string(), "no".to_string()]), + ..Default::default() + }, + CommonExt { + guided_grammar: Some("root ::= 'yes'".to_string()), + ..Default::default() + }, + ] + } + + #[test] + fn structured_response_format_rejects_explicit_guided_decoding() { + let response_format = Some(dynamo_protocols::types::ResponseFormat::JsonObject); + + for common in explicit_guided_decoding_options() { + let error = validate_response_format_conflicts(&response_format, &common).unwrap_err(); + assert_eq!( + error.to_string(), + "`response_format` cannot be used together with `guided_json`, `guided_regex`, \ + `guided_choice`, or `guided_grammar`" + ); + } + } + + #[test] + fn text_response_format_allows_explicit_guided_decoding() { + let response_format = Some(dynamo_protocols::types::ResponseFormat::Text); + + for common in explicit_guided_decoding_options() { + validate_response_format_conflicts(&response_format, &common).unwrap(); + } + } + + #[test] + fn empty_guided_choice_does_not_conflict_with_response_format() { + let response_format = Some(dynamo_protocols::types::ResponseFormat::JsonObject); + let common = CommonExt { + guided_choice: Some(Vec::new()), + ..Default::default() + }; + + validate_response_format_conflicts(&response_format, &common).unwrap(); + } + #[test] fn validate_chat_template_args_rejects_nested_chat_template() { let args = HashMap::from([( diff --git a/tests/frontend/vllm_prepost_worker.py b/tests/frontend/vllm_prepost_worker.py index fd4e95b931c2..d5745a4928c7 100644 --- a/tests/frontend/vllm_prepost_worker.py +++ b/tests/frontend/vllm_prepost_worker.py @@ -76,9 +76,7 @@ async def main(): runtime = DistributedRuntime(asyncio.get_running_loop(), "etcd", "tcp") endpoint = runtime.endpoint("test.vllm-prepost.generate") runtime_config = ModelRuntimeConfig() - runtime_config.set_structural_tag_mode("on") - runtime_config.set_structural_tag_scope("always") - runtime_config.set_structural_tag_schema("strict") + runtime_config.set_structural_tag({"scope": "always", "schema": "strict"}) await register_model( ModelInput.Tokens, ModelType.Chat,