From 8ef9302e15a9e94138c743c926f2c8a0b73348c6 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Fri, 21 Aug 2026 12:30:06 -0700 Subject: [PATCH 01/15] K3 structural tag default Signed-off-by: Indrajit Bhosale --- lib/llm/src/preprocessor/structural_tag.rs | 110 ++++++++++++++++++--- lib/llm/src/preprocessor/tool_choice.rs | 12 ++- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/lib/llm/src/preprocessor/structural_tag.rs b/lib/llm/src/preprocessor/structural_tag.rs index b16973bf7efa..e1ac1eabaee3 100644 --- a/lib/llm/src/preprocessor/structural_tag.rs +++ b/lib/llm/src/preprocessor/structural_tag.rs @@ -22,13 +22,16 @@ fn is_kimi_k2_parser(parser_name: Option<&str>) -> bool { fn requires_intrinsic_structural_tag(parser_name: Option<&str>, tool_choice: &ToolChoice) -> bool { // K2 forced calls and K3 named calls cannot use Dynamo's generic JSON-schema - // fallback because both families emit native, marker-delimited formats. - // Treat their structural tags as part of implementing these standard OpenAI - // request shapes, not as an operator opt-in. K3 required remains on its - // intentional prompt-level XTML path. + // fallback because both families emit native, marker-delimited formats. K3 + // auto also needs its model-native tag: the tag keeps the tools channel + // optional, but constrains a call after the model chooses to start one. + // Treat these tags as part of implementing the standard OpenAI request + // shapes, not as an operator opt-in. K3 required remains on its intentional + // prompt-level XTML path. (is_kimi_k2_parser(parser_name) && matches!(tool_choice, ToolChoice::Required | ToolChoice::Named(_))) - || (is_kimi_k3_parser(parser_name) && matches!(tool_choice, ToolChoice::Named(_))) + || (is_kimi_k3_parser(parser_name) + && matches!(tool_choice, ToolChoice::Auto | ToolChoice::Named(_))) } fn should_skip_tool_call_ban(exclude_tools_when_none: bool, tool_choice: &ToolChoice) -> bool { @@ -46,8 +49,9 @@ impl OpenAIPreprocessor { preprocessed_request: &mut PreprocessedRequest, ) -> Result { let parser_name = self.tool_call_parser.as_deref(); + let intrinsic_structural_tag = requires_intrinsic_structural_tag(parser_name, tool_choice); if self.runtime_config.structural_tag_mode == StructuralTagMode::Off - && !requires_intrinsic_structural_tag(parser_name, tool_choice) + && !intrinsic_structural_tag { return Ok(false); } @@ -81,12 +85,14 @@ impl OpenAIPreprocessor { return Self::apply_tool_call_ban(builder, preprocessed_request); } - if !Self::should_apply_tool_call_format( - self.runtime_config.structural_tag_scope, - tool_choice, - tools, - parallel_tool_calls, - ) { + if !intrinsic_structural_tag + && !Self::should_apply_tool_call_format( + self.runtime_config.structural_tag_scope, + tool_choice, + tools, + parallel_tool_calls, + ) + { return Ok(false); } @@ -318,7 +324,15 @@ mod tests { } #[test] - fn named_kimi_k3_is_intrinsic_even_when_global_mode_is_off() { + fn auto_and_named_kimi_k3_are_intrinsic_even_when_global_mode_is_off() { + assert!(requires_intrinsic_structural_tag( + Some("kimi_k3"), + &ToolChoice::Auto + )); + assert!(requires_intrinsic_structural_tag( + Some("kimi-k3"), + &ToolChoice::Auto + )); let named = ToolChoice::Named("get_weather".to_string()); assert!(requires_intrinsic_structural_tag(Some("kimi_k3"), &named)); assert!(requires_intrinsic_structural_tag(Some("kimi-k3"), &named)); @@ -433,6 +447,76 @@ mod tests { assert!(request.sampling_options.guided_decoding.is_none()); } + #[test] + fn kimi_k3_auto_installs_native_tag_when_global_mode_is_off() { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); + mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; + mdc.runtime_config.structural_tag_scope = StructuralTagScope::Auto; + mdc.runtime_config.tool_call_parser = Some("kimi_k3".to_string()); + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: Some(serde_json::json!({ + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"] + })), + // K3 auto must be constrained even when the caller does not opt in + // to OpenAI strict schema enforcement. + strict: None, + }]; + let mut request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_structural_tag(&ToolChoice::Auto, &tools, None, false, &mut request) + .unwrap(); + + assert!(applied); + let format = &request + .sampling_options + .guided_decoding + .as_ref() + .unwrap() + .structural_tag + .as_ref() + .unwrap()["format"]; + assert_eq!(format["type"], "triggered_tags"); + assert_eq!(format["at_least_one"], false); + assert_eq!( + format["triggers"], + serde_json::json!(["<|open|>tools<|sep|>"]) + ); + assert_eq!( + format["excludes"], + serde_json::json!([ + "<|open|>think<|sep|>", + "<|close|>think<|sep|>", + "<|open|>call" + ]) + ); + let tools_part = &format["tags"][0]; + assert_eq!(tools_part["begin"], "<|open|>tools<|sep|>"); + assert_eq!(tools_part["content"]["at_least_one"], true); + assert_eq!( + tools_part["content"]["tags"][0]["begin"], + "<|open|>call tool=\"get_weather\" index=\"" + ); + let arguments = &tools_part["content"]["tags"][0]["content"]["elements"][2]; + assert_eq!(arguments["type"], "sequence"); + assert_eq!( + arguments["elements"][0]["begin"], + "<|open|>argument key=\"location\" type=\"string\"<|sep|>" + ); + assert!( + arguments["elements"][0]["content"]["pattern"] + .as_str() + .is_some_and(|pattern| pattern.ends_with('+')), + "the required location argument must be non-empty" + ); + } + #[test] fn tool_choice_none_skips_ban_only_when_prompt_excludes_tools() { let tools = [ToolDefinition { diff --git a/lib/llm/src/preprocessor/tool_choice.rs b/lib/llm/src/preprocessor/tool_choice.rs index 630758d00537..64110bcbb096 100644 --- a/lib/llm/src/preprocessor/tool_choice.rs +++ b/lib/llm/src/preprocessor/tool_choice.rs @@ -48,11 +48,13 @@ impl OpenAIPreprocessor { /// Apply guided decoding for OpenAI tool-choice requests. /// /// Structural tags are preferred when enabled and supported by the configured - /// tool-call parser. Supported K2 forced requests and named K3 requests - /// intrinsically use their native structural tags because generic JSON cannot - /// represent their tool calls. Other forced choices fall back to the legacy - /// JSON-schema constraint when structural tags are not applied, except K3 - /// required requests, which stay on the prompt-level XTML path. + /// tool-call parser. Supported K2 forced requests and K3 auto/named requests + /// intrinsically use their native structural tags. K3 auto keeps tool use + /// optional while constraining any call the model elects to make; forced K2 + /// and named K3 calls cannot be represented by the generic JSON fallback. + /// Other forced choices fall back to that legacy JSON-schema constraint when + /// structural tags are not applied, except K3 required requests, which stay + /// on the prompt-level XTML path. pub(super) fn apply_tool_choice_guided_decoding( &self, request: &NvCreateChatCompletionRequest, From 0be3c9c0964af95600d90cf22f87b1682fa742bc Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 2 Sep 2026 22:31:38 -0700 Subject: [PATCH 02/15] Fix comments Signed-off-by: Indrajit Bhosale --- .../dynamo/common/configuration/groups/runtime_args.py | 7 ++++--- .../common/tests/configuration/test_runtime_args.py | 9 +++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/components/src/dynamo/common/configuration/groups/runtime_args.py index 632e70ce4a4a..e322b503251f 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/components/src/dynamo/common/configuration/groups/runtime_args.py @@ -306,9 +306,10 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: default="auto", choices=["auto", "strict"], help="Controls parameter schema strictness inside structural tags. " - "'auto': real schema only for tools with strict=true; " - "syntactically constrained but schema-unconstrained for all other tools. " - "'strict': real parameter schema for all tools.", + "'auto': declared parameter schema for tools with strict omitted or true; " + "tools with strict=false keep the native tool envelope constrained but use " + "schema-unconstrained arguments. " + "'strict': declared parameter schema for all tools, overriding strict=false.", ) add_argument( g, diff --git a/components/src/dynamo/common/tests/configuration/test_runtime_args.py b/components/src/dynamo/common/tests/configuration/test_runtime_args.py index 41627a143a10..f18a5fc177ba 100644 --- a/components/src/dynamo/common/tests/configuration/test_runtime_args.py +++ b/components/src/dynamo/common/tests/configuration/test_runtime_args.py @@ -42,6 +42,15 @@ def test_structural_tags_default_on_for_supported_parsers(monkeypatch): assert config.dyn_structural_tag_schema == "auto" +def test_structural_tag_schema_help_describes_omitted_strict(monkeypatch): + monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCHEMA", raising=False) + + _, help_text = _parse_runtime_args([]) + + assert "strict omitted or true" in help_text + assert "overriding strict=false" in help_text + + def test_structural_tag_global_opt_out(monkeypatch): monkeypatch.setenv("DYN_ENABLE_STRUCTURAL_TAG", "false") From a370f85480a549526c4bf2856bd88f257933a63d Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Tue, 8 Sep 2026 09:50:36 -0700 Subject: [PATCH 03/15] Add Structural on by deafult in dynamo Signed-off-by: Indrajit Bhosale --- .../configuration/groups/runtime_args.py | 5 +- components/src/dynamo/frontend/prepost.py | 69 +++++- .../src/dynamo/frontend/sglang_prepost.py | 174 +++++++++++---- .../src/dynamo/frontend/sglang_processor.py | 36 ++- .../dynamo/frontend/structural_tag_policy.py | 52 +++++ .../tests/test_sglang_processor_unit.py | 210 ++++++++++++++---- .../tests/test_structural_tag_policy.py | 73 ++++++ .../tests/test_vllm_processor_unit.py | 122 ++++++++-- .../src/dynamo/frontend/vllm_processor.py | 16 +- .../components/runtime-configuration.mdx | 2 +- .../structural-tag.md | 44 ++-- lib/backend-common/src/lib.rs | 1 - lib/bindings/python/rust/backend.rs | 4 +- lib/llm/src/discovery/worker_set.rs | 4 - lib/llm/src/http/service.rs | 10 +- lib/llm/src/local_model/runtime_config.rs | 26 --- lib/llm/src/preprocessor/structural_tag.rs | 65 +----- lib/llm/src/protocols/openai.rs | 11 - lib/sidecar/sglang/src/engine.rs | 28 +-- 19 files changed, 658 insertions(+), 294 deletions(-) create mode 100644 components/src/dynamo/frontend/structural_tag_policy.py create mode 100644 components/src/dynamo/frontend/tests/test_structural_tag_policy.py diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/components/src/dynamo/common/configuration/groups/runtime_args.py index e322b503251f..ceeb3705a9b8 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/components/src/dynamo/common/configuration/groups/runtime_args.py @@ -284,8 +284,9 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: env_var="DYN_ENABLE_STRUCTURAL_TAG", default=True, help="Enable structural tag guided decoding for tool calls when the configured " - "parser and backend support it. On the Rust frontend preprocessing path, " - "explicitly disabling this flag is authoritative.", + "parser and backend support it. Explicitly disabling this flag is " + "authoritative for Rust, Python vLLM, and Python SGLang frontend " + "preprocessing.", ) add_argument( g, diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index ad58f654ee80..1f8ec6d2a2e9 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -36,6 +36,15 @@ from dynamo.llm.exceptions import InvalidArgument from .thinking import apply_default_thinking_mode_to_template_kwargs +from .structural_tag_policy import ( + effective_tool_strict, + should_attempt_structural_tag, +) + +try: + from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag +except ImportError: # Older supported vLLM releases do not expose this registry. + get_model_structural_tag = None if TYPE_CHECKING: from vllm.config import ModelConfig @@ -181,21 +190,32 @@ def _should_build_tool_call_guidance( # enforced guarantee. sglang_prepost.py has the same gap. if tool_choice == "none": return False - if _is_forced_tool_choice(tool_choice): - return True - if structural_tag_mode != "on": - return False - if tool_choice != "auto": - return False - if structural_tag_scope == "always": - return True # An explicit single-call request is enough to attempt structural-tag # guidance. Forced-choice JSON guidance also bounds its array schema below. explicit_single_call = ( "parallel_tool_calls" in request.model_fields_set and request.parallel_tool_calls is False ) - return explicit_single_call or any(_tool_is_strict(tool) for tool in request.tools) + tool_choice_kind = ( + "required" + if tool_choice == "required" + else "named" + if _is_named_tool_choice(tool_choice) + else "auto" + if tool_choice == "auto" + else "other" + ) + attempt_structural_tag = should_attempt_structural_tag( + mode=structural_tag_mode, + scope=structural_tag_scope, + tool_choice_kind=tool_choice_kind, + has_tools=True, + any_explicit_strict=any(_tool_is_strict(tool) for tool in request.tools), + parallel_tool_calls_explicitly_false=explicit_single_call, + ) + # Forced choices retain the generic JSON fallback when structural tags are + # disabled or unavailable. + return attempt_structural_tag or _is_forced_tool_choice(tool_choice) def _request_for_vllm_structural_tag( @@ -203,13 +223,14 @@ def _request_for_vllm_structural_tag( *, structural_tag_schema: str, ) -> ChatCompletionRequest: - strict_schema = structural_tag_schema == "strict" tools = [ tool.model_copy( update={ "function": tool.function.model_copy( update={ - "strict": True if strict_schema else tool.function.strict, + "strict": effective_tool_strict( + tool.function.strict, structural_tag_schema + ), } ) } @@ -241,7 +262,31 @@ def build_tool_call_guided_decoding( request, structural_tag_schema=structural_tag_schema, ) - structural_tag = tool_parser.get_structural_tag(request_for_tag) + try: + structural_tag_model = getattr(tool_parser, "structural_tag_model", None) + structural_tag = ( + get_model_structural_tag( + model=structural_tag_model, + tools=request_for_tag.tools, + tool_choice=request_for_tag.tool_choice, + reasoning=False, + ) + if structural_tag_model is not None + and get_model_structural_tag is not None + else tool_parser.get_structural_tag(request_for_tag) + ) + except ( + AttributeError, + KeyError, + NotImplementedError, + RuntimeError, + TypeError, + ValueError, + ): + # Parser implementations are third-party capability providers. A + # failed structural-tag build must preserve the pre-existing JSON + # or unconstrained compatibility path for the request. + structural_tag = None if structural_tag is not None: tag_value = ( structural_tag.model_dump() diff --git a/components/src/dynamo/frontend/sglang_prepost.py b/components/src/dynamo/frontend/sglang_prepost.py index eeb830d6b9f0..90f9d88df78c 100644 --- a/components/src/dynamo/frontend/sglang_prepost.py +++ b/components/src/dynamo/frontend/sglang_prepost.py @@ -31,6 +31,10 @@ from dynamo.common.utils.engine_response import trailing_stop_prefix_len from dynamo.common.utils.guided_json import admits_only_empty_object +from .structural_tag_policy import ( + effective_tool_strict, + should_attempt_structural_tag, +) from .thinking import apply_default_thinking_mode_to_template_kwargs from .utils import PreprocessError, random_call_id @@ -198,13 +202,24 @@ def _client_wants_separate_reasoning(request: dict[str, Any]) -> bool: return bool(value) -def convert_tools(tools: list[dict[str, Any]] | None) -> list[SglangTool] | None: - """Convert OpenAI tool dicts to SGLang Tool objects.""" +def convert_tools( + tools: list[dict[str, Any]] | None, + *, + structural_tag_schema: str | None = None, +) -> list[SglangTool] | None: + """Convert OpenAI tool dicts to SGLang Tool objects. + + ``structural_tag_schema`` is set only for the guidance copy. Prompt and + parser copies retain SGLang's existing omitted-strict representation. + """ if not tools: return None sglang_tools = [] for tool in tools: func = tool.get("function", {}) + strict = func.get("strict", False) + if structural_tag_schema is not None: + strict = effective_tool_strict(func.get("strict"), structural_tag_schema) sglang_tools.append( SglangTool( type=tool.get("type", "function"), @@ -212,7 +227,7 @@ def convert_tools(tools: list[dict[str, Any]] | None) -> list[SglangTool] | None name=func.get("name", ""), description=func.get("description"), parameters=func.get("parameters"), - strict=func.get("strict", False), + strict=strict, ), ) ) @@ -561,11 +576,30 @@ def _call_with_optional_parallel_tool_calls( return func(*args) +def _call_structure_constraint( + func: Any, + *args: Any, + parallel_tool_calls: Any, + thinking_mode: bool, +) -> Any: + """Call SGLang parser APIs across supported signature versions.""" + kwargs: dict[str, Any] = {} + if _callable_accepts_kwarg(func, "parallel_tool_calls"): + kwargs["parallel_tool_calls"] = parallel_tool_calls + if _callable_accepts_kwarg(func, "thinking_mode"): + kwargs["thinking_mode"] = thinking_mode + return func(*args, **kwargs) + + def build_tool_call_guided_decoding( request: dict[str, Any], *, tool_call_parser_name: str | None, sglang_tools: list[SglangTool] | None, + structural_tag_mode: str = "off", + structural_tag_scope: str = "auto", + structural_tag_schema: str = "auto", + force_reasoning: bool = False, ) -> dict[str, Any] | None: """Build native-SGLang-like tool call constraints for guided decoding.""" if not sglang_tools: @@ -578,7 +612,74 @@ def build_tool_call_guided_decoding( parallel_tool_calls = request.get("parallel_tool_calls") constraint: Any = None - if tool_choice == "required" or _is_named_tool_choice(tool_choice): + is_named_choice = _is_named_tool_choice(tool_choice) + is_forced_choice = tool_choice == "required" or is_named_choice + sglang_tool_choice: Any = tool_choice + if is_named_choice: + sglang_tool_choice = SglangToolChoice( + type="function", + function=SglangToolChoiceFuncName( + name=tool_choice["function"]["name"], + ), + ) + tool_choice_kind = ( + "required" + if tool_choice == "required" + else "named" + if is_named_choice + else "auto" + if tool_choice == "auto" + else "other" + ) + raw_tools = request.get("tools") or [] + attempt_structural_tag = should_attempt_structural_tag( + mode=structural_tag_mode, + scope=structural_tag_scope, + tool_choice_kind=tool_choice_kind, + has_tools=bool(raw_tools), + any_explicit_strict=any( + tool.get("function", {}).get("strict") is True for tool in raw_tools + ), + parallel_tool_calls_explicitly_false=( + "parallel_tool_calls" in request and parallel_tool_calls is False + ), + ) + + if attempt_structural_tag and tool_call_parser_name: + guidance_tools = convert_tools( + raw_tools, + structural_tag_schema=structural_tag_schema, + ) + tool_call_parser_name = _normalize_sglang_parser_name(tool_call_parser_name) + try: + parser = FunctionCallParser( + tools=guidance_tools, + tool_call_parser=tool_call_parser_name, + ) + constraint = _call_structure_constraint( + parser.get_structure_constraint, + sglang_tool_choice, + parallel_tool_calls=parallel_tool_calls, + thinking_mode=force_reasoning, + ) + except ( + AttributeError, + KeyError, + NotImplementedError, + RuntimeError, + TypeError, + ValueError, + ): + # This is a third-party capability probe. Unsupported parser/schema + # combinations retain the existing forced-choice JSON fallback or, + # for auto, unconstrained generation. + constraint = None + + guided_decoding = _serialize_tool_constraint(constraint) + if guided_decoding is not None: + return guided_decoding + + if is_forced_choice: if named_closed_zero_arg_tool(request) is not None: return {"regex": r"\{\}"} @@ -586,14 +687,6 @@ def build_tool_call_guided_decoding( # ToolChoice) for the named-function case — passing our raw dict # would silently fall through and return None, disabling guided # decoding and letting the model omit required fields. - sglang_tool_choice: Any = tool_choice - if _is_named_tool_choice(tool_choice): - sglang_tool_choice = SglangToolChoice( - type="function", - function=SglangToolChoiceFuncName( - name=tool_choice["function"]["name"], - ), - ) constraint = ( "json_schema", _call_with_optional_parallel_tool_calls( @@ -603,42 +696,24 @@ def build_tool_call_guided_decoding( parallel_tool_calls=parallel_tool_calls, ), ) - # TODO: this applies a structural-tag constraint for tool_choice="auto" - # whenever a tool-call parser is configured, and reads NONE of - # structural_tag_mode / structural_tag_scope / structural_tag_schema. Those - # knobs are accepted on the CLI and published into the model card by - # sglang/register.py, so an operator setting the mode to "off" still gets a - # constraint on this path. prepost.py requires structural_tag_mode == "on" - # (_should_build_tool_call_guidance) and preprocessor/structural_tag.rs - # requires StructuralTagMode != Off, so with an explicit mode of "off" the - # same request is unconstrained on both of those paths and constrained here. - # Also unlike them, "auto" is never gated on scope/strict, so this behaves as - # scope="always" with no way to narrow it. - elif tool_call_parser_name: - tool_call_parser_name = _normalize_sglang_parser_name(tool_call_parser_name) - parser = FunctionCallParser( - tools=sglang_tools, - tool_call_parser=tool_call_parser_name, - ) - constraint = _call_with_optional_parallel_tool_calls( - parser.get_structure_constraint, - tool_choice, - parallel_tool_calls=parallel_tool_calls, - ) + return _serialize_tool_constraint(constraint) - if isinstance(constraint, tuple) and len(constraint) == 2: - if constraint[0] == "json_schema": - return {"json": constraint[1]} - if constraint[0] == "structural_tag": - tag_value = constraint[1] - # SGLang returns a Pydantic model (LegacyStructuralTagResponseFormat) - # here. Convert to a plain dict before it hits the RPC layer — - # msgpack/serde_json cannot serialize BaseModel instances. - if hasattr(tag_value, "model_dump"): - tag_value = tag_value.model_dump() - return {"structural_tag": tag_value} - return None +def _serialize_tool_constraint(constraint: Any) -> dict[str, Any] | None: + if not (isinstance(constraint, tuple) and len(constraint) == 2): + return None + if constraint[0] == "json_schema": + return {"json": constraint[1]} + if constraint[0] != "structural_tag": + return None + + tag_value = constraint[1] + # SGLang returns a Pydantic model (LegacyStructuralTagResponseFormat) + # here. Convert to a plain dict before it hits the RPC layer because + # msgpack/serde_json cannot serialize BaseModel instances. + if hasattr(tag_value, "model_dump"): + tag_value = tag_value.model_dump() + return {"structural_tag": tag_value} def build_response_format_guided_decoding( @@ -735,6 +810,9 @@ def preprocess_chat_request( exclude_tools_when_tool_choice_none: bool = True, template_force_reasoning: bool = False, default_thinking_mode: str | None = None, + structural_tag_mode: str = "off", + structural_tag_scope: str = "auto", + structural_tag_schema: str = "auto", ) -> SglangPreprocessResult: """Preprocess a chat request using SGLang tokenizer and parser APIs. @@ -834,6 +912,10 @@ def preprocess_chat_request( request, tool_call_parser_name=tool_call_parser_name, sglang_tools=sglang_tools, + structural_tag_mode=structural_tag_mode, + structural_tag_scope=structural_tag_scope, + structural_tag_schema=structural_tag_schema, + force_reasoning=force_reasoning, ) # TODO: response_format wins here even when tool_choice is "required" or names # a function, so a request that demanded a tool call can come back with none diff --git a/components/src/dynamo/frontend/sglang_processor.py b/components/src/dynamo/frontend/sglang_processor.py index e5d3781e6f92..d2afa7d47d3e 100644 --- a/components/src/dynamo/frontend/sglang_processor.py +++ b/components/src/dynamo/frontend/sglang_processor.py @@ -38,6 +38,7 @@ detect_force_reasoning_from_template, preprocess_chat_request, ) +from .structural_tag_policy import runtime_structural_tag_options from .thinking import runtime_default_thinking_mode from .utils import ( PreprocessError, @@ -251,6 +252,9 @@ def _map_finish_reason(raw: str | None) -> str | None: _w_exclude_tools_when_tool_choice_none: bool = True _w_template_force_reasoning: bool = False _w_default_thinking_mode: str | None = None +_w_structural_tag_mode: str = "off" +_w_structural_tag_scope: str = "auto" +_w_structural_tag_schema: str = "auto" def _load_chat_template(chat_template: str | None) -> str | None: @@ -302,11 +306,15 @@ def _init_worker( template_force_reasoning: bool = False, chat_template: str | None = None, default_thinking_mode: str | None = None, + structural_tag_mode: str = "off", + structural_tag_scope: str = "auto", + structural_tag_schema: str = "auto", ) -> None: """Initialize a worker process with its own tokenizer.""" global _w_tokenizer, _w_tool_call_parser_name, _w_reasoning_parser_name global _w_exclude_tools_when_tool_choice_none, _w_template_force_reasoning - global _w_default_thinking_mode + global _w_default_thinking_mode, _w_structural_tag_mode + global _w_structural_tag_scope, _w_structural_tag_schema _w_tokenizer = _load_tokenizer(model_path, trust_remote_code) if chat_template is not None: _w_tokenizer.chat_template = chat_template @@ -315,6 +323,9 @@ def _init_worker( _w_exclude_tools_when_tool_choice_none = exclude_tools_when_tool_choice_none _w_template_force_reasoning = template_force_reasoning _w_default_thinking_mode = default_thinking_mode + _w_structural_tag_mode = structural_tag_mode + _w_structural_tag_scope = structural_tag_scope + _w_structural_tag_schema = structural_tag_schema def _preprocess_worker( @@ -331,6 +342,9 @@ def _preprocess_worker( exclude_tools_when_tool_choice_none=_w_exclude_tools_when_tool_choice_none, template_force_reasoning=_w_template_force_reasoning, default_thinking_mode=_w_default_thinking_mode, + structural_tag_mode=_w_structural_tag_mode, + structural_tag_scope=_w_structural_tag_scope, + structural_tag_schema=_w_structural_tag_schema, ) n = request.get("n", 1) @@ -481,6 +495,9 @@ def __init__( preprocess_workers: int = 0, stream_interval: int = 1, default_thinking_mode: str | None = None, + structural_tag_mode: str = "off", + structural_tag_scope: str = "auto", + structural_tag_schema: str = "auto", ): self.tokenizer = tokenizer # Detect force_reasoning once from the chat template, matching @@ -505,6 +522,9 @@ def __init__( self.debug_perf = debug_perf self.stream_interval = stream_interval self.default_thinking_mode = default_thinking_mode + self.structural_tag_mode = structural_tag_mode + self.structural_tag_scope = structural_tag_scope + self.structural_tag_schema = structural_tag_schema self.preprocess_pool = preprocess_pool if preprocess_pool is not None: self._worker_semaphore: asyncio.Semaphore | None = asyncio.Semaphore( @@ -562,6 +582,9 @@ async def _generator_inner( exclude_tools_when_tool_choice_none=self.exclude_tools_when_tool_choice_none, template_force_reasoning=self.template_force_reasoning, default_thinking_mode=self.default_thinking_mode, + structural_tag_mode=self.structural_tag_mode, + structural_tag_scope=self.structural_tag_scope, + structural_tag_schema=self.structural_tag_schema, ) if self.debug_perf: @@ -996,6 +1019,11 @@ async def chat_engine_factory( or _runtime_config_parser_name(mdc, "reasoning_parser") ) default_thinking_mode = runtime_default_thinking_mode(mdc.runtime_config()) + ( + structural_tag_mode, + structural_tag_scope, + structural_tag_schema, + ) = runtime_structural_tag_options(mdc.runtime_config()) if tool_call_parser_name: logger.info("SGLang tool call parser: %s", tool_call_parser_name) @@ -1024,6 +1052,9 @@ async def chat_engine_factory( template_force_reasoning, chat_template, default_thinking_mode, + structural_tag_mode, + structural_tag_scope, + structural_tag_schema, ), ) futures = [ @@ -1060,6 +1091,9 @@ async def chat_engine_factory( preprocess_workers=preprocess_workers, stream_interval=self.stream_interval, default_thinking_mode=default_thinking_mode, + structural_tag_mode=structural_tag_mode, + structural_tag_scope=structural_tag_scope, + structural_tag_schema=structural_tag_schema, ) gen.exclude_tools_when_tool_choice_none = ( self.config.exclude_tools_when_tool_choice_none diff --git a/components/src/dynamo/frontend/structural_tag_policy.py b/components/src/dynamo/frontend/structural_tag_policy.py new file mode 100644 index 000000000000..b54b7499eaca --- /dev/null +++ b/components/src/dynamo/frontend/structural_tag_policy.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from typing import Any, Literal + +ToolChoiceKind = Literal["none", "auto", "required", "named", "other"] + + +def runtime_structural_tag_options( + runtime_config: Any, +) -> tuple[str, str, str]: + """Read structural-tag policy from a model-card runtime config.""" + if not isinstance(runtime_config, dict): + return "off", "auto", "auto" + return ( + runtime_config.get("structural_tag_mode", "off"), + runtime_config.get("structural_tag_scope", "auto"), + runtime_config.get("structural_tag_schema", "auto"), + ) + + +def should_attempt_structural_tag( + *, + mode: str, + scope: str, + tool_choice_kind: ToolChoiceKind, + has_tools: bool, + any_explicit_strict: bool, + parallel_tool_calls_explicitly_false: bool, +) -> bool: + """Return whether frontend preprocessing should request a structural tag.""" + if mode != "on" or not has_tools or tool_choice_kind == "none": + return False + if tool_choice_kind in {"required", "named"}: + return True + if tool_choice_kind != "auto": + return False + if scope == "always": + return True + return any_explicit_strict or parallel_tool_calls_explicitly_false + + +def effective_tool_strict(request_strict: bool | None, schema_mode: str) -> bool: + """Resolve request strictness for grammar construction. + + In ``auto`` schema mode, omitted ``strict`` is schema-enforced and only an + explicit ``strict: false`` opts out. ``strict`` schema mode overrides that + request-level opt-out. + """ + return schema_mode == "strict" or request_strict is not False diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index a9afcda2b395..9715499a5b44 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -1276,6 +1276,8 @@ def test_shared_tool_guidance_policy(self, tokenizer, case): tokenizer=tokenizer, tool_call_parser_name="kimi_k2", reasoning_parser_name=None, + structural_tag_mode="on", + structural_tag_scope="always", ) assert ( @@ -1318,17 +1320,16 @@ def test_none_when_tool_choice_none(self): ) def test_auto_tool_guidance_normalizes_minimax_m3_alias(self, monkeypatch): - tools = convert_tools( - [ - { - "type": "function", - "function": { - "name": "get_weather", - "parameters": {"type": "object", "properties": {}}, - }, - } - ] - ) + raw_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": {"type": "object", "properties": {}}, + }, + } + ] + tools = convert_tools(raw_tools) seen = {} # Test double: capture the parser name used for guided decoding setup. @@ -1347,9 +1348,11 @@ def get_structure_constraint(self, tool_choice, **kwargs): ) guided = build_tool_call_guided_decoding( - {"tool_choice": "auto"}, + {"tool_choice": "auto", "tools": raw_tools}, tool_call_parser_name="minimax_m3_nom", sglang_tools=tools, + structural_tag_mode="on", + structural_tag_scope="always", ) assert seen["tool_call_parser"] == "minimax-m3" @@ -1466,21 +1469,20 @@ def old_get_json_schema_constraint(sglang_tools, tool_choice): def test_auto_tool_choice_supports_older_structure_constraint_signature( self, monkeypatch ): - tools = convert_tools( - [ - { - "type": "function", - "function": { - "name": "get_weather", - "strict": True, - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - }, + raw_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, }, - } - ] - ) + }, + } + ] + tools = convert_tools(raw_tools) class OldFunctionCallParser: def __init__(self, *, tools, tool_call_parser): @@ -1498,40 +1500,156 @@ def get_structure_constraint(self, tool_choice): ) guided = build_tool_call_guided_decoding( - {"tool_choice": "auto", "parallel_tool_calls": False}, + { + "tool_choice": "auto", + "parallel_tool_calls": False, + "tools": raw_tools, + }, tool_call_parser_name="kimi_k2", sglang_tools=tools, + structural_tag_mode="on", ) assert guided == {"structural_tag": {"type": "object"}} def test_auto_strict_tools_can_build_structural_tag_guidance(self): - tools = convert_tools( - [ - { - "type": "function", - "function": { - "name": "get_weather", - "strict": True, - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, + raw_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "strict": True, + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], }, - } - ] - ) + }, + } + ] + tools = convert_tools(raw_tools) guided = build_tool_call_guided_decoding( - {"tool_choice": "auto"}, + {"tool_choice": "auto", "tools": raw_tools}, tool_call_parser_name="kimi_k2", sglang_tools=tools, + structural_tag_mode="on", + structural_tag_scope="auto", ) assert isinstance(guided, dict) assert "structural_tag" in guided + @pytest.mark.parametrize( + ("request_strict", "include_strict", "schema_mode", "expected_strict"), + [ + (None, False, "auto", True), + (True, True, "auto", True), + (False, True, "auto", False), + (False, True, "strict", True), + ], + ) + def test_structural_tag_guidance_uses_effective_schema_policy( + self, + monkeypatch, + request_strict, + include_strict, + schema_mode, + expected_strict, + ): + function = { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + }, + } + if include_strict: + function["strict"] = request_strict + raw_tools = [{"type": "function", "function": function}] + seen = {} + + class CapturingFunctionCallParser: + def __init__(self, *, tools, tool_call_parser): + del tool_call_parser + seen["strict"] = tools[0].function.strict + + def get_structure_constraint( + self, tool_choice, *, parallel_tool_calls=None, thinking_mode=False + ): + del tool_choice, parallel_tool_calls + seen["thinking_mode"] = thinking_mode + return "structural_tag", {"type": "object"} + + monkeypatch.setattr( + sglang_prepost_module, + "FunctionCallParser", + CapturingFunctionCallParser, + ) + + guided = build_tool_call_guided_decoding( + {"tool_choice": "auto", "tools": raw_tools}, + tool_call_parser_name="kimi_k2", + sglang_tools=convert_tools(raw_tools), + structural_tag_mode="on", + structural_tag_scope="always", + structural_tag_schema=schema_mode, + force_reasoning=True, + ) + + assert guided == {"structural_tag": {"type": "object"}} + assert seen == {"strict": expected_strict, "thinking_mode": True} + assert function.get("strict") is request_strict + + def test_mode_off_does_not_probe_auto_structural_tag(self, monkeypatch): + raw_tools = [parity_tool()] + + class UnexpectedFunctionCallParser: + def __init__(self, **kwargs): + raise AssertionError(f"unexpected structural-tag probe: {kwargs}") + + monkeypatch.setattr( + sglang_prepost_module, + "FunctionCallParser", + UnexpectedFunctionCallParser, + ) + + assert ( + build_tool_call_guided_decoding( + {"tool_choice": "auto", "tools": raw_tools}, + tool_call_parser_name="kimi_k2", + sglang_tools=convert_tools(raw_tools), + structural_tag_mode="off", + structural_tag_scope="always", + ) + is None + ) + + def test_builder_error_uses_required_json_fallback(self, monkeypatch): + raw_tools = [parity_tool()] + + class RaisingFunctionCallParser: + def __init__(self, **kwargs): + del kwargs + raise ValueError("unsupported schema") + + monkeypatch.setattr( + sglang_prepost_module, + "FunctionCallParser", + RaisingFunctionCallParser, + ) + + guided = build_tool_call_guided_decoding( + {"tool_choice": "required", "tools": raw_tools}, + tool_call_parser_name="kimi_k2", + sglang_tools=convert_tools(raw_tools), + structural_tag_mode="on", + structural_tag_scope="always", + ) + + assert guided is not None + assert set(guided) == {"json"} + def test_tool_parser_requires_tools(self): """Tool parser is not created if no tools in request.""" tcp, rp = create_parsers( @@ -2273,9 +2391,15 @@ class FakeTokenizer: exclude_tools_when_tool_choice_none=True, chat_template="custom template", default_thinking_mode="disabled", + structural_tag_mode="on", + structural_tag_scope="always", + structural_tag_schema="strict", ) assert sglang_processor_module._w_tokenizer.chat_template == "custom template" assert sglang_processor_module._w_default_thinking_mode == "disabled" + assert sglang_processor_module._w_structural_tag_mode == "on" + assert sglang_processor_module._w_structural_tag_scope == "always" + assert sglang_processor_module._w_structural_tag_schema == "strict" def test_with_reasoning_parser(self, tokenizer): """Reasoning parser is attached to result.""" diff --git a/components/src/dynamo/frontend/tests/test_structural_tag_policy.py b/components/src/dynamo/frontend/tests/test_structural_tag_policy.py new file mode 100644 index 000000000000..ce93dd3073c6 --- /dev/null +++ b/components/src/dynamo/frontend/tests/test_structural_tag_policy.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from dynamo.frontend.structural_tag_policy import ( + effective_tool_strict, + runtime_structural_tag_options, + should_attempt_structural_tag, +) + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + + +@pytest.mark.parametrize( + ("mode", "scope", "choice", "strict", "single", "expected"), + [ + ("off", "always", "auto", False, False, False), + ("on", "always", "auto", False, False, True), + ("on", "auto", "auto", True, False, True), + ("on", "auto", "auto", False, True, True), + ("on", "auto", "auto", False, False, False), + ("on", "auto", "required", False, False, True), + ("on", "auto", "named", False, False, True), + ("on", "always", "none", True, True, False), + ], +) +def test_structural_tag_activation_policy( + mode, scope, choice, strict, single, expected +): + assert ( + should_attempt_structural_tag( + mode=mode, + scope=scope, + tool_choice_kind=choice, + has_tools=True, + any_explicit_strict=strict, + parallel_tool_calls_explicitly_false=single, + ) + is expected + ) + + +@pytest.mark.parametrize( + ("request_strict", "schema_mode", "expected"), + [ + (None, "auto", True), + (True, "auto", True), + (False, "auto", False), + (False, "strict", True), + ], +) +def test_effective_tool_strict(request_strict, schema_mode, expected): + assert effective_tool_strict(request_strict, schema_mode) is expected + + +def test_runtime_structural_tag_options_are_conservative_when_absent(): + assert runtime_structural_tag_options(None) == ("off", "auto", "auto") + assert runtime_structural_tag_options({}) == ("off", "auto", "auto") + + +def test_runtime_structural_tag_options_read_model_card_policy(): + assert runtime_structural_tag_options( + { + "structural_tag_mode": "on", + "structural_tag_scope": "always", + "structural_tag_schema": "strict", + } + ) == ("on", "always", "strict") diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index d8fc87da63e0..5efe6b062689 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -2307,8 +2307,9 @@ def test_strict_schema_uses_copy(self, tokenizer): assert request.tools[0].function.strict is None assert parser.requests[0].messages is request.messages - # Auto schema mode must preserve an omitted strict flag as unset. - def test_auto_schema_preserves_unset_strict(self, tokenizer): + # In auto schema mode, omitted strict is schema-enforced. Only an explicit + # strict:false opts out. + def test_auto_schema_enforces_omitted_strict(self, tokenizer): parser = _FakeStructuralTagParser() guided = build_tool_call_guided_decoding( self._request(tokenizer), @@ -2318,7 +2319,106 @@ def test_auto_schema_preserves_unset_strict(self, tokenizer): structural_tag_schema="auto", ) - assert guided == {"structural_tag": {"format": {"strict": [None]}}} + assert guided == {"structural_tag": {"format": {"strict": [True]}}} + + def test_auto_schema_preserves_explicit_strict_false(self, tokenizer): + parser = _FakeStructuralTagParser() + request = self._request( + tokenizer, + tools=[ + { + **TOOL_REQUEST["tools"][0], + "function": { + **TOOL_REQUEST["tools"][0]["function"], + "strict": False, + }, + } + ], + ) + + guided = build_tool_call_guided_decoding( + request, + parser, + structural_tag_mode="on", + structural_tag_scope="always", + structural_tag_schema="auto", + ) + + assert guided == {"structural_tag": {"format": {"strict": [False]}}} + + def test_structural_tag_builder_error_uses_forced_json_fallback(self, tokenizer): + class RaisingParser(_FakeStructuralTagParser): + def get_structural_tag(self, request, *, reasoning=False): + del request, reasoning + raise ValueError("unsupported schema") + + guided = build_tool_call_guided_decoding( + self._request(tokenizer, tool_choice="required"), + RaisingParser(), + structural_tag_mode="on", + structural_tag_scope="always", + ) + + assert guided is not None + assert set(guided) == {"json"} + + def test_declared_model_uses_registry_without_vllm_env_gate( + self, tokenizer, monkeypatch + ): + seen = {} + + class RegistryParser: + structural_tag_model = "qwen_3_coder" + + def get_structural_tag(self, request): + raise AssertionError(f"unexpected parser-level env gate: {request}") + + def fake_get_model_structural_tag(*, model, tools, tool_choice, reasoning): + seen.update( + model=model, + strict=tools[0].function.strict, + tool_choice=tool_choice, + reasoning=reasoning, + ) + return _FakeStructuralTag({"format": {"type": "tag"}}) + + monkeypatch.setattr( + prepost_module, + "get_model_structural_tag", + fake_get_model_structural_tag, + ) + + guided = build_tool_call_guided_decoding( + self._request(tokenizer), + RegistryParser(), + structural_tag_mode="on", + structural_tag_scope="always", + ) + + assert guided == {"structural_tag": {"format": {"type": "tag"}}} + assert seen == { + "model": "qwen_3_coder", + "strict": True, + "tool_choice": "auto", + "reasoning": False, + } + + def test_missing_registry_uses_parser_method(self, tokenizer, monkeypatch): + class CompatibleParser(_FakeStructuralTagParser): + structural_tag_model = "qwen_3_coder" + + parser = CompatibleParser() + monkeypatch.setattr(prepost_module, "get_model_structural_tag", None) + + guided = build_tool_call_guided_decoding( + self._request(tokenizer), + parser, + structural_tag_mode="on", + structural_tag_scope="always", + ) + + assert guided == {"structural_tag": {"format": {"strict": [True]}}} + assert len(parser.requests) == 1 # Parser-created grammar must survive preprocessing as guided decoding. @pytest.mark.asyncio @@ -3016,22 +3116,6 @@ def test_runtime_config_context_length(vllm_processor_module, runtime_config, ex assert vllm_processor_module._runtime_config_context_length(mdc) == expected -def test_runtime_config_structural_tag_options(vllm_processor_module): - mdc = SimpleNamespace( - runtime_config=lambda: { - "structural_tag_mode": "on", - "structural_tag_scope": "always", - "structural_tag_schema": "strict", - } - ) - - assert vllm_processor_module._runtime_config_structural_tag_options(mdc) == ( - "on", - "always", - "strict", - ) - - # Regression: MistralTokenizer (--tokenizer-mode mistral) has no chat_template # attribute; a direct read used to crash vLLM preprocessing. diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index 5bb5a229535e..16f22c508b24 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -43,6 +43,7 @@ from dynamo.vllm.errors import vllm_client_error_to_http_error from .prepost import StreamingPostProcessor, preprocess_chat_request +from .structural_tag_policy import runtime_structural_tag_options from .thinking import runtime_default_thinking_mode from .utils import ( as_error_envelope, @@ -142,19 +143,6 @@ def _runtime_config_context_length(mdc: ModelDeploymentCard) -> int | None: return context_length -def _runtime_config_structural_tag_options( - mdc: ModelDeploymentCard, -) -> tuple[str, str, str]: - runtime_config = mdc.runtime_config() - if not isinstance(runtime_config, dict): - return "off", "auto", "auto" - return ( - runtime_config.get("structural_tag_mode", "off"), - runtime_config.get("structural_tag_scope", "auto"), - runtime_config.get("structural_tag_schema", "auto"), - ) - - def _ensure_chat_template( tokenizer: Any, local_dir: str, chat_template_flag: str | None ) -> None: @@ -1233,7 +1221,7 @@ async def chat_engine_factory( structural_tag_mode, structural_tag_scope, structural_tag_schema, - ) = _runtime_config_structural_tag_options(mdc) + ) = runtime_structural_tag_options(mdc.runtime_config()) block_size = self.config.kv_cache_block_size or 16 diff --git a/docs/fern/pages/reference/components/runtime-configuration.mdx b/docs/fern/pages/reference/components/runtime-configuration.mdx index 4bbee1aaf23b..06871092cdcd 100644 --- a/docs/fern/pages/reference/components/runtime-configuration.mdx +++ b/docs/fern/pages/reference/components/runtime-configuration.mdx @@ -167,7 +167,7 @@ Unless a field is marked environment-only, it has both a CLI flag and an environ - Enable structural tag guided decoding for tool calls when the configured parser and backend support it. On the Rust frontend preprocessing path, explicitly setting this option to `false` is authoritative: Dynamo does not inject a structural tag and uses the parser's compatibility fallback. Legacy inline Python preprocessing paths retain their existing engine-specific constraint behavior. + Enable structural tag guided decoding for tool calls when the configured parser and backend support it. The Rust, Python vLLM, and Python SGLang frontend preprocessing paths use this setting for `auto`, required, and named tool choices. Explicitly setting it to `false` is authoritative: Dynamo does not inject a tool-call structural tag and uses the existing compatibility fallback. Native Rust sidecars retain their conservative `off`/`auto` behavior and are not included in the default change. Environment variable: `DYN_ENABLE_STRUCTURAL_TAG` diff --git a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md index e6e349cda4a3..b8e12c6ca0a2 100644 --- a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md +++ b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md @@ -25,7 +25,7 @@ Benefits: ## Prerequisites -- A backend engine and transport with structural-tag/xgrammar support. +- A backend engine that accepts structural-tag/xgrammar guided decoding. - A Dynamo tool call parser that provides a structural tag config (see [Supported Parsers](#supported-parsers) below). @@ -80,15 +80,15 @@ decoding. See [Activation Scope](#activation-scope) for the exact policy. | Flag | Values | Default | Description | |---|---|---|---| -| `--dyn-enable-structural-tag` | bool | `true` | Master switch. On the Rust frontend preprocessing path, explicitly disabling it prevents all structural-tag injection. | +| `--dyn-enable-structural-tag` | bool | `true` | Master switch for the Rust, Python vLLM, and Python SGLang frontend preprocessing paths. Explicitly disabling it prevents tool-call structural-tag injection. | | `--dyn-structural-tag-scope` | `auto`, `always` | `always` | Controls when structural tags are activated (see [Activation Scope](#activation-scope)). | | `--dyn-structural-tag-schema` | `auto`, `strict` | `auto` | Controls parameter schema strictness inside structural tags (see [Schema Modes](#schema-modes)). | ## Supported Parsers -Not all parsers support structural tags. Parsers without a structural tag -config fall back to standard behaviour (a diagnostic is logged if structural -tags are enabled but the parser does not support them). +Not all parsers support structural tags. Parsers without a structural-tag +builder fall back to their existing best-effort tool-calling behavior without +rejecting the request. Currently tested and supported: @@ -100,15 +100,16 @@ Currently tested and supported: Contributions adding structural tag support for new parsers are welcome. -Parser support and backend support are separate. If the parser has no builder, -or the backend transport reports that it cannot carry structural tags, Dynamo -uses the existing best-effort tool-calling path instead of rejecting the -request. +This list describes Dynamo's Rust parser registry. The Python vLLM and SGLang +frontend processors apply the same mode, scope, and schema policy through the +tool parser supplied by their installed engine version. Parser availability can +therefore differ by backend and engine version. > [!NOTE] -> The activation and fallback policy on this page describes Dynamo's Rust -> frontend preprocessing path. Legacy inline Python preprocessing paths retain -> their existing engine-specific constraint behavior. +> Native Rust sidecars retain their existing conservative `off`/`auto` +> settings and are not included in this default change. The default-on policy +> applies when regular vLLM or SGLang workers publish the deployment runtime +> configuration consumed by frontend preprocessing. ## Activation Scope @@ -121,7 +122,7 @@ based on the request's `tool_choice`: |---|---| | `required` / `named` | Always | | `auto` | Always | -| `none` | Exclusion tag only | +| `none` | Exclusion tag on the Rust path only | ### `auto` (legacy conditional activation) @@ -129,7 +130,7 @@ based on the request's `tool_choice`: |---|---| | `required` / `named` | Always | | `auto` | Only when any tool has `strict: true` or `parallel_tool_calls` is `false` | -| `none` | Exclusion tag only (bans tool call tokens, see [below](#tool_choicenone-and-token-banning)) | +| `none` | Exclusion tag on the Rust path only (see [below](#tool_choicenone-and-token-banning)) | ## Schema Modes @@ -149,14 +150,19 @@ tool arguments inside the structural tag: flag. If a model-native builder cannot safely represent part of a schema, it keeps -the strongest safe tool envelope and relaxes that argument section. Dynamo logs -the compatibility fallback rather than introducing a new request error. +the strongest safe tool envelope and relaxes that argument section. If the +builder cannot produce a structural tag at all, Dynamo uses the existing +compatibility path rather than introducing a new request error; automatic tool +choice may therefore remain unconstrained for that parser/schema combination. ## `tool_choice="none"` and Token Banning -When `tool_choice="none"` and structural tags are enabled, Dynamo injects an -exclusion structural tag that bans parser-specific tool-call start tokens (for -example ``) so the model cannot start native tool-call syntax. +On the Rust frontend preprocessing path, when `tool_choice="none"` and +structural tags are enabled, Dynamo injects an exclusion structural tag that +bans parser-specific tool-call start tokens (for example ``) so the +model cannot start native tool-call syntax. The Python vLLM and SGLang frontend +processors continue to handle `none` through their existing prompt and response +shaping; this release does not add token banning to those paths. **Quality trade-off**. If tools remain in the prompt on `none` (often via `--no-exclude-tools-when-tool-choice-none` to keep the chat prefix stable for KV diff --git a/lib/backend-common/src/lib.rs b/lib/backend-common/src/lib.rs index b7f4de57af75..f7a51b8ffbc7 100644 --- a/lib/backend-common/src/lib.rs +++ b/lib/backend-common/src/lib.rs @@ -31,7 +31,6 @@ pub mod worker; pub use args::CommonArgs; pub use disagg::DisaggregationMode; -pub use dynamo_llm::local_model::runtime_config::TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY; pub use dynamo_llm::model_type::ModelInput; pub use engine::{ AsyncEngineContext, BootstrapInfo, CompletionUsage, ComponentSnapshot, EngineConfig, diff --git a/lib/bindings/python/rust/backend.rs b/lib/bindings/python/rust/backend.rs index 42e2c738dbd3..8ba566c20b74 100644 --- a/lib/bindings/python/rust/backend.rs +++ b/lib/bindings/python/rust/backend.rs @@ -413,8 +413,8 @@ impl WorkerConfig { runtime = None, disaggregation_mode = DisaggregationMode::Aggregated, health_check_payload = None, - structural_tag_mode = "on".to_string(), - structural_tag_scope = "always".to_string(), + structural_tag_mode = "off".to_string(), + structural_tag_scope = "auto".to_string(), structural_tag_schema = "auto".to_string(), route_to_encoder = false, media_decoder = None, diff --git a/lib/llm/src/discovery/worker_set.rs b/lib/llm/src/discovery/worker_set.rs index 5efac29dcb47..47345c7e013a 100644 --- a/lib/llm/src/discovery/worker_set.rs +++ b/lib/llm/src/discovery/worker_set.rs @@ -356,10 +356,6 @@ impl WorkerSet { crate::protocols::openai::ParsingOptions { structural_tag_mode: self.card.runtime_config.structural_tag_mode, structural_tag_scope: self.card.runtime_config.structural_tag_scope, - structural_tag_supported: self - .card - .runtime_config - .tool_call_structural_tag_supported(), exclude_tools_when_tool_choice_none: self .card .runtime_config diff --git a/lib/llm/src/http/service.rs b/lib/llm/src/http/service.rs index ca8c6f8b52e5..886075e8c944 100644 --- a/lib/llm/src/http/service.rs +++ b/lib/llm/src/http/service.rs @@ -71,7 +71,6 @@ fn apply_request_tool_call_parsing_options( parsing_options.structural_tag_mode, parsing_options.structural_tag_scope, parsing_options.exclude_tools_when_tool_choice_none, - parsing_options.structural_tag_supported, )? .is_required(); let guided_tool_constraint = crate::preprocessor::tool_choice::guided_tool_constraint( @@ -197,11 +196,12 @@ mod tests { ); } - // With the operator default (`structural_tag_mode = Off`), the same non-Kimi - // parser must NOT get a structural tag — confirms the mode gate above is real, - // not a permanently-on regression. + // With the low-level ParsingOptions default (`structural_tag_mode = Off`), + // the same non-Kimi parser must NOT get a structural tag. Regular workers + // explicitly publish the deployment's default-on policy into the model card; + // native sidecars that do not do so retain this conservative default. #[test] - fn operator_default_mode_off_does_not_resolve_structural_tag_for_a_non_kimi_parser() { + fn parsing_options_default_mode_off_does_not_resolve_structural_tag() { let parsing_options = ParsingOptions { tool_call_parser: Some("qwen3_coder".to_string()), ..Default::default() diff --git a/lib/llm/src/local_model/runtime_config.rs b/lib/llm/src/local_model/runtime_config.rs index 060fd3910db6..bdb47efb7eda 100644 --- a/lib/llm/src/local_model/runtime_config.rs +++ b/lib/llm/src/local_model/runtime_config.rs @@ -40,15 +40,6 @@ pub const TOKEN_BUDGET_RUNTIME_KEY: &str = "token_budget"; pub const TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY: &str = "tool_call_structural_tag_excludes_reasoning"; -/// Runtime-data key indicating whether the backend transport accepts structural -/// tag guided decoding. -/// -/// Absence means `true` for compatibility with workers that predate this -/// capability. Backends that cannot represent structural tags must publish -/// `false` so the frontend can preserve its legacy fallback path. -pub const TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY: &str = - "tool_call_structural_tag_supported"; - /// Describes which request-token overflows the frontend may reject early. /// /// The combined limit already accounts for engine-reserved tokens. A false @@ -653,23 +644,6 @@ impl ModelRuntimeConfig { } } - /// Whether the backend transport can carry structural-tag guided decoding. - /// Missing metadata remains enabled for compatibility with older workers. - pub(crate) fn tool_call_structural_tag_supported(&self) -> bool { - match self.get_engine_specific::(TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY) { - Ok(Some(supported)) => supported, - Ok(None) => true, - Err(error) => { - tracing::warn!( - %error, - key = TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY, - "Invalid structural-tag capability metadata; using compatibility fallback" - ); - false - } - } - } - pub fn effective_tokenizer_backend(&self) -> TokenizerBackend { self.tokenizer_backend .unwrap_or_else(TokenizerBackend::from_env_or_default) diff --git a/lib/llm/src/preprocessor/structural_tag.rs b/lib/llm/src/preprocessor/structural_tag.rs index 1dab10405082..c257efb5ca73 100644 --- a/lib/llm/src/preprocessor/structural_tag.rs +++ b/lib/llm/src/preprocessor/structural_tag.rs @@ -67,7 +67,7 @@ fn should_skip_tool_call_ban(exclude_tools_when_none: bool, tool_choice: &ToolCh /// /// This is the single owner of "is a structural tag applicable and available" — /// covering the operator's global `structural_tag_mode`/`structural_tag_scope`, -/// backend transport support, the `tool_choice=None` ban-tag exclusion, and real +/// the `tool_choice=None` ban-tag exclusion, and real /// parser-registry builder availability. A caller can only /// reach `Required` by way of a real, registered /// [`dynamo_parsers::tool_calling::StructuralTagBuilder`] — it cannot ask for the @@ -98,7 +98,6 @@ impl StructuralTagDecision { /// (`http::service::apply_request_tool_call_parsing_options`) must propagate /// this error rather than falling back to `NotApplicable`, or an invalid forced /// choice would silently install a structural tag anyway. -#[allow(clippy::too_many_arguments)] pub(crate) fn structural_tag_decision( parser_name: Option<&str>, tool_choice: &ToolChoice, @@ -107,13 +106,12 @@ pub(crate) fn structural_tag_decision( mode: StructuralTagMode, scope: StructuralTagScope, exclude_tools_when_tool_choice_none: bool, - backend_supports_structural_tags: bool, ) -> Result { - // Validate before any policy or capability gate. Kimi K3 required requests use a + // Validate before any policy gate. Kimi K3 required requests use a // prompt-level XTML path, but they still need an actual tool to require. validate_forced_tool_choice(tool_choice, tools)?; - if mode == StructuralTagMode::Off || !backend_supports_structural_tags { + if mode == StructuralTagMode::Off { return Ok(StructuralTagDecision::NotApplicable); } @@ -174,7 +172,6 @@ impl OpenAIPreprocessor { self.runtime_config.structural_tag_mode, self.runtime_config.structural_tag_scope, self.runtime_config.exclude_tools_when_tool_choice_none, - self.runtime_config.tool_call_structural_tag_supported(), )? else { return Ok(false); @@ -323,7 +320,6 @@ mod tests { use std::{path::PathBuf, sync::Arc}; use crate::{ - local_model::runtime_config::TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY, model_card::ModelDeploymentCard, protocols::common::{OutputOptions, SamplingOptions, StopConditions}, }; @@ -685,61 +681,6 @@ mod tests { )); } - #[test] - fn backend_capability_false_skips_structural_tag() { - let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); - let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::On; - mdc.runtime_config.structural_tag_scope = StructuralTagScope::Always; - mdc.runtime_config.tool_call_parser = Some("qwen3_coder".to_string()); - mdc.runtime_config - .set_engine_specific(TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY, false) - .unwrap(); - let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); - let tools = [ToolDefinition { - name: "get_weather".to_string(), - parameters: None, - strict: None, - }]; - let mut request = preprocessed_request(); - - let applied = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::Auto, &tools, None, false, &mut request) - .unwrap(); - - assert!(!applied); - assert!(request.sampling_options.guided_decoding.is_none()); - } - - #[test] - fn malformed_backend_capability_skips_structural_tag() { - let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); - let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::On; - mdc.runtime_config.structural_tag_scope = StructuralTagScope::Always; - mdc.runtime_config.tool_call_parser = Some("qwen3_coder".to_string()); - mdc.runtime_config.runtime_data.insert( - TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY.to_string(), - serde_json::json!("not-a-boolean"), - ); - let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); - let tools = [ToolDefinition { - name: "get_weather".to_string(), - parameters: None, - strict: None, - }]; - let mut request = preprocessed_request(); - - let applied = preprocessor - .apply_tool_choice_structural_tag(&ToolChoice::Auto, &tools, None, false, &mut request) - .unwrap(); - - assert!(!applied); - assert!(request.sampling_options.guided_decoding.is_none()); - } - #[test] fn tool_choice_none_skips_ban_only_when_prompt_excludes_tools() { let tools = [ToolDefinition { diff --git a/lib/llm/src/protocols/openai.rs b/lib/llm/src/protocols/openai.rs index 7fc074680434..a594a3c595b1 100644 --- a/lib/llm/src/protocols/openai.rs +++ b/lib/llm/src/protocols/openai.rs @@ -368,10 +368,6 @@ impl GuidedToolConstraint { } } -const fn default_structural_tag_supported() -> bool { - true -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ParsingOptions { pub tool_call_parser: Option, @@ -418,12 +414,6 @@ pub struct ParsingOptions { #[serde(default)] pub structural_tag_scope: crate::local_model::runtime_config::StructuralTagScope, - /// Whether the selected backend transport accepts structural-tag guidance. - /// Defaults to true for compatibility with workers that predate capability - /// publication. - #[serde(default = "default_structural_tag_supported")] - pub structural_tag_supported: bool, - #[serde( default = "crate::local_model::runtime_config::default_exclude_tools_when_tool_choice_none" )] @@ -458,7 +448,6 @@ impl ParsingOptions { move_reasoning_to_content_when_empty: false, structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::default(), structural_tag_scope: crate::local_model::runtime_config::StructuralTagScope::default(), - structural_tag_supported: true, exclude_tools_when_tool_choice_none: crate::local_model::runtime_config::default_exclude_tools_when_tool_choice_none(), tools: Vec::new(), diff --git a/lib/sidecar/sglang/src/engine.rs b/lib/sidecar/sglang/src/engine.rs index 738a152b6a0e..fd007c5fe291 100644 --- a/lib/sidecar/sglang/src/engine.rs +++ b/lib/sidecar/sglang/src/engine.rs @@ -11,7 +11,7 @@ use async_trait::async_trait; use dynamo_backend_common::{ AsyncEngineContext, DisaggregationMode, DynamoError, EngineConfig, GenerateContext, KvEventSource, LLMEngine, LLMEngineOutput, LLMEngineOutputExt, LlmRegistration, ModelInput, - PreprocessedRequest, TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY, WorkerConfig, usage, + PreprocessedRequest, WorkerConfig, usage, }; use dynamo_sidecar_common::{GrpcEndpoint, GrpcTransportConfig, SidecarStartupError}; use futures::stream::BoxStream; @@ -825,11 +825,6 @@ fn build_engine_config( "grpc_service".to_string(), Value::String("sglang.runtime.v1.SglangService".to_string()), ); - runtime_data.insert( - TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY.to_string(), - Value::Bool(false), - ); - Ok(EngineConfig { model: discovery.model_path.clone(), served_model_name: discovery.served_model_name.clone(), @@ -855,8 +850,7 @@ mod tests { use serde_json::json; use super::{ - DisaggregationMode, DiscoveredKvEventSource, Discovery, - TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY, build_engine_config, + DisaggregationMode, DiscoveredKvEventSource, Discovery, build_engine_config, discover_kv_event_sources, resolve_bootstrap_host_with_local, }; @@ -883,24 +877,6 @@ mod tests { assert_eq!(host.as_deref(), Some("prefill.example")); } - #[test] - fn native_grpc_advertises_structural_tags_as_unsupported() { - let config = build_engine_config( - &discovery(json!({})), - DisaggregationMode::Aggregated, - None, - None, - ) - .unwrap(); - - assert_eq!( - config - .runtime_data - .get(TOOL_CALL_STRUCTURAL_TAG_SUPPORTED_RUNTIME_KEY), - Some(&json!(false)) - ); - } - #[test] fn server_host_precedes_dist_init_addr() { let host = resolve_bootstrap_host_with_local( From 1650c7d227c83165bccabc32c96409159b4277a0 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 9 Sep 2026 16:12:13 -0700 Subject: [PATCH 04/15] Pre-Commit fix Signed-off-by: Indrajit Bhosale --- components/src/dynamo/frontend/prepost.py | 5 +---- components/src/dynamo/frontend/sglang_prepost.py | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index 97c27ab30a76..e37a02574c69 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -35,10 +35,7 @@ from dynamo.common.utils.guided_json import admits_only_empty_object from dynamo.llm.exceptions import InvalidArgument -from .structural_tag_policy import ( - effective_tool_strict, - should_attempt_structural_tag, -) +from .structural_tag_policy import effective_tool_strict, should_attempt_structural_tag from .thinking import apply_default_thinking_mode_to_template_kwargs from .utils import legacy_guided_decoding diff --git a/components/src/dynamo/frontend/sglang_prepost.py b/components/src/dynamo/frontend/sglang_prepost.py index 0e6bd7dba8c7..e1b6c4bf45ec 100644 --- a/components/src/dynamo/frontend/sglang_prepost.py +++ b/components/src/dynamo/frontend/sglang_prepost.py @@ -32,10 +32,7 @@ from dynamo.common.utils.guided_json import admits_only_empty_object from dynamo.llm.exceptions import InvalidArgument -from .structural_tag_policy import ( - effective_tool_strict, - should_attempt_structural_tag, -) +from .structural_tag_policy import effective_tool_strict, should_attempt_structural_tag from .thinking import apply_default_thinking_mode_to_template_kwargs from .utils import PreprocessError, legacy_guided_decoding, random_call_id From 18678e90b7e9c124bad0dbf657eafabe351c64cd Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Tue, 22 Sep 2026 17:07:11 -0700 Subject: [PATCH 05/15] Fix coderabbit commetns Signed-off-by: Indrajit Bhosale --- .../tests/configuration/test_runtime_args.py | 1 + lib/llm/src/preprocessor/tool_choice.rs | 57 ++++++++++++------- 2 files changed, 36 insertions(+), 22 deletions(-) diff --git a/components/src/dynamo/common/tests/configuration/test_runtime_args.py b/components/src/dynamo/common/tests/configuration/test_runtime_args.py index 82e436ed3e2e..d1a011815c35 100644 --- a/components/src/dynamo/common/tests/configuration/test_runtime_args.py +++ b/components/src/dynamo/common/tests/configuration/test_runtime_args.py @@ -34,6 +34,7 @@ def _parse_runtime_args(argv: list[str]) -> tuple[DynamoRuntimeConfig, str]: def test_structural_tags_default_on_for_supported_parsers(monkeypatch): monkeypatch.delenv("DYN_ENABLE_STRUCTURAL_TAG", raising=False) monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCOPE", raising=False) + monkeypatch.delenv("DYN_STRUCTURAL_TAG_SCHEMA", raising=False) config, _ = _parse_runtime_args([]) diff --git a/lib/llm/src/preprocessor/tool_choice.rs b/lib/llm/src/preprocessor/tool_choice.rs index 9649192872ef..2f5ca4669cf8 100644 --- a/lib/llm/src/preprocessor/tool_choice.rs +++ b/lib/llm/src/preprocessor/tool_choice.rs @@ -234,8 +234,13 @@ pub(crate) fn guided_tool_constraint( if has_explicit_guided_decoding(request) { return Ok(GuidedToolConstraint::None); } - // K3 forced requests are served by a prompt-level XTML instruction; no JSON schema. - if uses_kimi_k3_parser(tool_call_parser, reasoning_parser) { + let parser_tool_choice = convert_tool_choice(tool_choice); + // Kimi forced requests use marker-delimited native formats that generic JSON + // guidance cannot represent. Keep this compatibility path aligned with + // request preprocessing for either configured parser slot. + if requires_native_tool_call_format(tool_call_parser, &parser_tool_choice) + || requires_native_tool_call_format(reasoning_parser, &parser_tool_choice) + { return Ok(GuidedToolConstraint::None); } // Validate the forced choice against the actual tools the same way @@ -254,15 +259,6 @@ pub(crate) fn guided_tool_constraint( } } -/// True when either configured parser is Kimi K3. -/// -/// K3 forced requests are served by a prompt-level XTML instruction, so they must -/// NOT be reported as guided JSON even though their `tool_choice` is forced. -fn uses_kimi_k3_parser(tool_call_parser: Option<&str>, reasoning_parser: Option<&str>) -> bool { - let is_k3 = |parser: &str| matches!(parser, "kimi_k3" | "kimi-k3"); - tool_call_parser.is_some_and(is_k3) || reasoning_parser.is_some_and(is_k3) -} - /// Map a forced `tool_choice` onto the guided-JSON parser constraint for its grammar. /// /// Only reachable once `get_tool_choice_guidance_from_tools` produced either JSON or @@ -501,18 +497,35 @@ mod tests { } #[test] - fn kimi_k3_is_detected_from_either_parser_slot() { - assert!(uses_kimi_k3_parser(Some("kimi_k3"), None)); - assert!(uses_kimi_k3_parser(Some("kimi-k3"), None)); - assert!(uses_kimi_k3_parser(None, Some("kimi_k3"))); - assert!(uses_kimi_k3_parser(None, Some("kimi-k3"))); - } + fn native_format_forced_choices_install_no_compatibility_json_constraint() { + let requests = [ + request(json!({ + "tools": tools(), + "tool_choice": "required" + })), + request(json!({ + "tools": tools(), + "tool_choice": { + "type": "function", + "function": {"name": "get_weather"} + } + })), + ]; - #[test] - fn non_k3_parsers_are_not_mistaken_for_k3() { - assert!(!uses_kimi_k3_parser(None, None)); - assert!(!uses_kimi_k3_parser(Some("kimi_k2"), Some("qwen3"))); - assert!(!uses_kimi_k3_parser(Some("qwen3_coder"), None)); + for request in &requests { + for (tool_call_parser, reasoning_parser) in [ + (Some("kimi_k2"), None), + (None, Some("kimi_k2")), + (Some("kimi_k3"), None), + (None, Some("kimi-k3")), + ] { + assert_eq!( + guided_tool_constraint(request, tool_call_parser, reasoning_parser, false,) + .expect("native-format choice is valid"), + GuidedToolConstraint::None, + ); + } + } } #[test] From 79a5fc8a503de198366782a46a31dbe49be3b769 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 30 Sep 2026 20:08:30 -0700 Subject: [PATCH 06/15] Fix comments Signed-off-by: Indrajit Bhosale --- .../configuration/groups/runtime_args.py | 6 +- components/src/dynamo/frontend/prepost.py | 8 + .../src/dynamo/frontend/sglang_prepost.py | 35 ++-- .../src/dynamo/frontend/sglang_processor.py | 3 + .../tests/test_sglang_processor_unit.py | 197 +++++++++++++++++- .../tests/test_vllm_processor_unit.py | 80 +++++++ .../components/runtime-configuration.mdx | 6 +- .../structural-tag.md | 31 ++- lib/llm/src/http/service.rs | 7 +- lib/llm/src/preprocessor/structural_tag.rs | 78 +++++-- lib/llm/src/preprocessor/tool_choice.rs | 81 ++++++- 11 files changed, 464 insertions(+), 68 deletions(-) diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/components/src/dynamo/common/configuration/groups/runtime_args.py index 311f2e4f849d..14c967a2b012 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/components/src/dynamo/common/configuration/groups/runtime_args.py @@ -295,9 +295,9 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: env_var="DYN_ENABLE_STRUCTURAL_TAG", default=True, help="Enable structural tag guided decoding for tool calls when the configured " - "parser and backend support it. Explicitly disabling this flag is " - "authoritative for Rust, Python vLLM, and Python SGLang frontend " - "preprocessing.", + "parser and backend support it. Disabling this flag suppresses optional " + "guidance; Rust retains native tags for Kimi K2 required/named and " + "Kimi K3 named tool choices. Python vLLM and SGLang respect the opt-out.", ) add_argument( g, diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index 91370a89deba..24778e44ffab 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -289,12 +289,20 @@ def build_tool_call_guided_decoding( ) try: structural_tag_model = getattr(tool_parser, "structural_tag_model", None) + structural_tag_kwargs = {} + if structural_tag_model == "hy_v4" and get_model_structural_tag is not None: + # HYV4's parser detects checkpoint-specific tokens at construction. + # Keep its suffix while Dynamo owns activation of the registry path. + structural_tag_kwargs["token_suffix"] = getattr( + tool_parser, "_extractor" + ).token_suffix structural_tag = ( get_model_structural_tag( model=structural_tag_model, tools=request_for_tag.tools, tool_choice=request_for_tag.tool_choice, reasoning=False, + **structural_tag_kwargs, ) if structural_tag_model is not None and get_model_structural_tag is not None diff --git a/components/src/dynamo/frontend/sglang_prepost.py b/components/src/dynamo/frontend/sglang_prepost.py index 23737e57ed95..c42ada78faa8 100644 --- a/components/src/dynamo/frontend/sglang_prepost.py +++ b/components/src/dynamo/frontend/sglang_prepost.py @@ -296,6 +296,7 @@ def create_parsers( reasoning_parser_name: str | None, sglang_tools: list[SglangTool] | None = None, force_reasoning: bool = False, + guided_decoding: dict[str, Any] | None = None, ) -> tuple[ToolCallParserType | None, ReasoningParser | None]: """Create tool call and reasoning parsers for a request. @@ -305,10 +306,9 @@ def create_parsers( If ``sglang_tools`` is provided, reuses them; otherwise converts from the request's ``tools`` field. - For ``tool_choice="required"`` or a named function, uses - :class:`JsonArrayParser` (matching native SGLang) since guided decoding - constrains the output to a JSON array. Otherwise uses the model-specific - :class:`FunctionCallParser`. + Required and named choices use the model-specific :class:`FunctionCallParser` + when the effective guidance is a structural tag. Their JSON fallback keeps + :class:`JsonArrayParser`. Automatic choices use :class:`FunctionCallParser`. """ if sglang_tools is None: sglang_tools = convert_tools(request.get("tools")) @@ -316,7 +316,9 @@ def create_parsers( tool_call_parser: ToolCallParserType | None = None if sglang_tools and tool_choice != "none": - if tool_choice == "required" or _is_named_tool_choice(tool_choice): + if (tool_choice == "required" or _is_named_tool_choice(tool_choice)) and not ( + guided_decoding is not None and "structural_tag" in guided_decoding + ): tool_call_parser = JsonArrayParser() elif tool_call_parser_name: tool_call_parser_name = _normalize_sglang_parser_name(tool_call_parser_name) @@ -691,7 +693,9 @@ def build_tool_call_guided_decoding( parser.get_structure_constraint, sglang_tool_choice, parallel_tool_calls=parallel_tool_calls, - thinking_mode=force_reasoning, + # The backend's reasoning gate consumes the reasoning prefix; + # this grammar starts with the tool payload after that boundary. + thinking_mode=False, ) except ( AttributeError, @@ -942,15 +946,6 @@ def preprocess_chat_request( tokenizer.apply_chat_template(template_messages, **template_kwargs) ) - # Build parsers after rendering, so DeepSeek-V4 can use its custom encoder - # while still sharing the existing Dynamo parser/guided-decoding behavior. - tool_call_parser, reasoning_parser = create_parsers( - request, - tool_call_parser_name=tool_call_parser_name, - reasoning_parser_name=effective_reasoning_parser_name, - sglang_tools=sglang_tools, - force_reasoning=force_reasoning, - ) response_format_guided_decoding = build_response_format_guided_decoding(request) tool_call_guided_decoding = build_tool_call_guided_decoding( request, @@ -1006,6 +1001,16 @@ def preprocess_chat_request( or tool_call_guided_decoding ) + # Match the parser to the installed format, including forced-choice fallback. + tool_call_parser, reasoning_parser = create_parsers( + request, + tool_call_parser_name=tool_call_parser_name, + reasoning_parser_name=effective_reasoning_parser_name, + sglang_tools=sglang_tools, + force_reasoning=force_reasoning, + guided_decoding=guided_decoding, + ) + return SglangPreprocessResult( prompt_token_ids=prompt_token_ids, tool_call_parser=tool_call_parser, diff --git a/components/src/dynamo/frontend/sglang_processor.py b/components/src/dynamo/frontend/sglang_processor.py index 409de3ddf688..4a22b381eb81 100644 --- a/components/src/dynamo/frontend/sglang_processor.py +++ b/components/src/dynamo/frontend/sglang_processor.py @@ -716,6 +716,9 @@ async def _generator_inner_pool( tool_call_parser_name=self.tool_call_parser_name, reasoning_parser_name=preproc_result.effective_reasoning_parser_name, force_reasoning=preproc_result.force_reasoning, + guided_decoding=preproc_result.dynamo_preproc["sampling_options"][ + "guided_decoding" + ], ) post = SglangStreamingPostProcessor( diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index 5588b2d392fe..80dfded40ae0 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -19,6 +19,7 @@ from enum import IntEnum import pytest +import xgrammar as xgr from _routed_engine_fakes import FakeRoutedEngine, FakeRoutedItem from _thinking_parity import RESOLVED_DISABLED, RESOLVED_ENABLED, THINKING_PARITY_CASES from _tool_guidance_parity import ( @@ -28,6 +29,8 @@ parity_tool, tool_choice_value, ) +from sglang.srt.constrained.reasoner_grammar_backend import ReasonerGrammarObject +from sglang.srt.constrained.xgrammar_backend import XGrammarGrammar from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.utils.hf_transformers_utils import get_tokenizer @@ -92,6 +95,13 @@ def tokenizer(): return get_tokenizer(MODEL) +@pytest.fixture(scope="module") +def tool_grammar_compiler(tokenizer): + return xgr.GrammarCompiler( + xgr.TokenizerInfo.from_huggingface(tokenizer), max_threads=1 + ) + + @pytest.fixture(scope="module") def byte_fallback_tokenizer(): return get_tokenizer(BYTE_FALLBACK_MODEL) @@ -1194,6 +1204,10 @@ def apply_chat_template(self, messages, **kwargs): sglang_processor_module, "_w_exclude_tools_when_tool_choice_none", True ) monkeypatch.setattr(sglang_processor_module, "_w_template_force_reasoning", False) + # Exercise enabled guidance independently of worker initialization or test order. + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_mode", "on") + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_scope", "always") + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_schema", "auto") request = { "model": "moonshotai/Kimi-K3", @@ -1482,6 +1496,183 @@ def test_structural_tag_builds_guided_decoding(self): ) == {"structural_tag": response_format} +@pytest.mark.core +@pytest.mark.timeout(60) +@pytest.mark.parametrize("use_pool", [False, True], ids=["inline", "pool"]) +@pytest.mark.parametrize( + "tool_choice", + [ + pytest.param("required", id="required"), + pytest.param( + {"type": "function", "function": {"name": "get_weather"}}, id="named" + ), + ], +) +def test_forced_structural_tag_returns_tool_call( + tokenizer, monkeypatch, use_pool, tool_choice +): + """Native forced-tool guidance must be decoded into an API tool call.""" + request = { + "model": MODEL, + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "chat_template_kwargs": {"enable_thinking": False}, + "stream": True, + "tool_choice": tool_choice, + "tools": [ + { + "type": "function", + "function": { + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } + ], + } + native_call = ( + "\n\n" + "\nParis\n\n" + "\n" + ) + engine = FakeRoutedEngine( + items=[{"token_ids": tokenizer.encode(native_call), "finish_reason": "stop"}] + ) + + if use_pool: + # Exercise the real worker and parser reconstruction without a subprocess. + monkeypatch.setattr(sglang_processor_module, "_w_tokenizer", tokenizer) + monkeypatch.setattr( + sglang_processor_module, "_w_tool_call_parser_name", "qwen3_coder" + ) + monkeypatch.setattr(sglang_processor_module, "_w_reasoning_parser_name", None) + monkeypatch.setattr( + sglang_processor_module, "_w_exclude_tools_when_tool_choice_none", True + ) + monkeypatch.setattr( + sglang_processor_module, "_w_template_force_reasoning", False + ) + monkeypatch.setattr(sglang_processor_module, "_w_default_thinking_mode", None) + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_mode", "on") + monkeypatch.setattr( + sglang_processor_module, "_w_structural_tag_scope", "always" + ) + monkeypatch.setattr(sglang_processor_module, "_w_structural_tag_schema", "auto") + + with ThreadPoolExecutor(max_workers=1) if use_pool else nullcontext() as pool: + processor = SglangProcessor( + tokenizer=tokenizer, + routed_engine=engine, + tool_call_parser_name="qwen3_coder", + reasoning_parser_name=None, + eos_token_ids=None, + preprocess_pool=pool, + preprocess_workers=1 if use_pool else 0, + structural_tag_mode="on", + structural_tag_scope="always", + ) + + async def collect(): + return [item async for item in processor.generator(request)] + + output = asyncio.run(collect()) + + # Check the actual installed guidance, not just the requested policy. + assert "structural_tag" in engine.requests[0]["sampling_options"]["guided_decoding"] + choices = [choice for item in output for choice in item["data"]["choices"]] + calls = [ + call for choice in choices for call in choice["delta"].get("tool_calls", []) + ] + assert calls, output + assert {call["index"] for call in calls} == {0} + assert [ + call["function"]["name"] for call in calls if call["function"].get("name") + ] == ["get_weather"] + assert json.loads( + "".join(call["function"].get("arguments", "") for call in calls) + ) == {"city": "Paris"} + assert not any(choice["delta"].get("content") for choice in choices) + assert choices[-1]["finish_reason"] == "tool_calls" + + +@pytest.mark.core +@pytest.mark.timeout(60) +@pytest.mark.parametrize("tool_choice", ["auto", "required", "named"]) +@pytest.mark.parametrize("thinking", [False, True], ids=["chat", "thinking"]) +@pytest.mark.parametrize( + "separate_reasoning", [False, True], ids=["combined", "separate"] +) +def test_tool_structural_tag_enforced_after_one_reasoning_boundary( + tokenizer, tool_grammar_compiler, tool_choice, thinking, separate_reasoning +): + """The backend consumes reasoning; the inner grammar must enforce tools next.""" + request = { + "model": MODEL, + "messages": [{"role": "user", "content": "Weather in Paris?"}], + "tools": [parity_tool()], + "tool_choice": tool_choice_value(tool_choice), + "chat_template_kwargs": {"enable_thinking": thinking}, + "separate_reasoning": separate_reasoning, + } + pre = preprocess_chat_request( + request, + tokenizer=tokenizer, + tool_call_parser_name="qwen3_coder", + reasoning_parser_name="qwen3", + structural_tag_mode="on", + structural_tag_scope="always", + ) + assert pre.force_reasoning is thinking + require_reasoning = _guided_output_requires_reasoning( + request, pre.force_reasoning, "qwen3", pre.guided_decoding + ) + assert require_reasoning is thinking + assert pre.guided_decoding is not None + compiled = tool_grammar_compiler.compile_structural_tag( + pre.guided_decoding["structural_tag"] + ) + + def grammar_after_reasoning(): + inner = XGrammarGrammar( + matcher=xgr.GrammarMatcher(compiled), + vocab_size=compiled.tokenizer_info.vocab_size, + ctx=compiled, + override_stop_tokens=None, + ) + grammar = ReasonerGrammarObject( + inner, think_end_ids=tokenizer.encode("", add_special_tokens=False) + ) + grammar.maybe_init_reasoning(require_reasoning) + if thinking: + for token in tokenizer.encode( + "I should check the weather.", add_special_tokens=False + ): + grammar.accept_token(token) + # The wrapper consumes the prefix, including the end marker. + assert inner.accepted_tokens == [] + return grammar + + native_call = ( + "\n\n" + "\nParis\n\n" + "\n" + ) + grammar = grammar_after_reasoning() + for token in tokenizer.encode(native_call, add_special_tokens=False): + grammar.accept_token(token) + grammar.accept_token(tokenizer.eos_token_id) + assert grammar.is_terminated() + + # A second reasoning prefix must not leave an undeclared tool unconstrained. + grammar = grammar_after_reasoning() + unknown_call = native_call.replace("get_weather", "undeclared_tool") + with pytest.raises(ValueError, match="Tokens not accepted"): + for token in tokenizer.encode(unknown_call, add_special_tokens=False): + grammar.accept_token(token) + + class TestBuildToolCallGuidedDecoding: # FRONTEND.3 — guided-decoding setup for tool_choice # Keep SGLang's guidance decisions aligned with the shared backend matrix. @pytest.mark.parametrize( @@ -1840,7 +2031,7 @@ def get_structure_constraint( assert guided == {"structural_tag": {"type": "object"}} assert seen == { "strict": expected_strict, - "thinking_mode": True, + "thinking_mode": False, "strict_level": "FUNCTION", } assert function.get("strict") is request_strict @@ -1964,7 +2155,7 @@ def test_both_parsers(self): assert rp is not None def test_required_creates_json_array_parser(self): - """tool_choice='required' creates JsonArrayParser, not FunctionCallParser.""" + """Required choice without structural-tag guidance keeps JsonArrayParser.""" request = { "tools": [ { @@ -1984,7 +2175,7 @@ def test_required_creates_json_array_parser(self): assert isinstance(tcp, JsonArrayParser) def test_named_tool_choice_creates_json_array_parser(self): - """Named tool_choice creates JsonArrayParser.""" + """Named choice without structural-tag guidance keeps JsonArrayParser.""" request = { "tools": [ { diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index 1464d93c8bd1..1f1c6139c36e 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -2463,6 +2463,86 @@ def test_auto_schema_preserves_explicit_strict_false(self, tokenizer): assert guided == {"structural_tag": {"format": {"strict": [False]}}} + @pytest.mark.parametrize( + "token_suffix", [":6124c78e", ""], ids=["suffixed", "plain"] + ) + @pytest.mark.parametrize("tool_choice_kind", ["auto", "required", "named"]) + @pytest.mark.parametrize("stream_response", [False, True], ids=["batch", "stream"]) + def test_hyv4_structural_tag_matches_checkpoint_tokens( + self, tokenizer, monkeypatch, token_suffix, tool_choice_kind, stream_response + ): + import vllm.envs as vllm_envs + import xgrammar as xgr + from vllm.tool_parsers.hy_v4_tool_parser import HYV4ToolParser + + # Dynamo's explicit policy must work without vLLM's environment gate. + monkeypatch.setattr(vllm_envs, "VLLM_ENFORCE_STRICT_TOOL_CALLING", False) + request = self._request( + tokenizer, + tools=[parity_tool()], + tool_choice=tool_choice_value(tool_choice_kind), + ) + markers = [ + f"<{closing}{name}{token_suffix}>" + for name in ("tool_calls", "tool_call", "arg_key", "arg_value") + for closing in ("", "/") + ] + # Only the vocabulary is needed to detect this checkpoint's suffix. + vocab = {marker: index for index, marker in enumerate(markers)} + parser = HYV4ToolParser( + SimpleNamespace(get_vocab=lambda: vocab, init_kwargs={}), request.tools + ) + assert parser.get_structural_tag(request) is None + + guided = build_tool_call_guided_decoding( + request, + parser, + structural_tag_mode="on", + structural_tag_scope="always", + ) + assert guided is not None + assert set(guided) == {"structural_tag"} + compiler = xgr.GrammarCompiler( + xgr.TokenizerInfo(["x"], vocab_type=xgr.VocabType.RAW), max_threads=1 + ) + compiled = compiler.compile_structural_tag(json.dumps(guided["structural_tag"])) + native_output = ( + f"get_weather" + f"city" + f"Paris" + f"" + ) + assert xgr.GrammarMatcher(compiled).accept_string(native_output) + assert not xgr.GrammarMatcher(compiled).accept_string( + native_output.replace("get_weather", "unknown_tool") + ) + + post = StreamingPostProcessor( + tokenizer=tokenizer, + request_for_sampling=request, + sampling_params=SamplingParams(), + prompt_token_ids=[], + tool_parser=parser, + reasoning_parser_class=None, + chat_template_kwargs={}, + stream_response=stream_response, + ) + choice = post.process_output( + SimpleNamespace( + index=0, + text=native_output, + token_ids=list(vocab.values()), + finish_reason="stop", + logprobs=None, + ) + ) + assert choice is not None + assert choice["finish_reason"] == "tool_calls" + assert not choice["delta"].get("content") + function = choice["delta"]["tool_calls"][0]["function"] + assert function["name"] == "get_weather" + assert json.loads(function["arguments"]) == {"city": "Paris"} + def test_structural_tag_builder_error_uses_forced_json_fallback(self, tokenizer): class RaisingParser(_FakeStructuralTagParser): def get_structural_tag(self, request, *, reasoning=False): diff --git a/docs/fern/pages/reference/components/runtime-configuration.mdx b/docs/fern/pages/reference/components/runtime-configuration.mdx index 7268fb9ac479..995bfc5720a3 100644 --- a/docs/fern/pages/reference/components/runtime-configuration.mdx +++ b/docs/fern/pages/reference/components/runtime-configuration.mdx @@ -263,13 +263,13 @@ Unless a field is marked environment-only, it has both a CLI flag and an environ - Enable structural tag guided decoding for tool calls when the configured parser and backend support it. The Rust, Python vLLM, and Python SGLang frontend preprocessing paths use this setting for `auto`, required, and named tool choices. Explicitly setting it to `false` is authoritative: Dynamo does not inject a tool-call structural tag and uses the existing compatibility fallback. Native Rust sidecars retain their conservative `off`/`auto` behavior and are not included in the default change. + Enable structural tag guided decoding for tool calls when the configured parser and backend support it. The Rust, Python vLLM, and Python SGLang frontend preprocessing paths use this setting for `auto`, required, and named tool choices. Setting it to `false` disables optional guidance. Rust preserves native tags for Kimi K2 required/named and Kimi K3 named choices; Python vLLM and SGLang respect the opt-out for all tool choices. Native Rust sidecars retain their conservative `off`/`auto` behavior, including this Kimi exception, and are not included in the default change. Environment variable: `DYN_ENABLE_STRUCTURAL_TAG` - Controls when structural tags are activated. `always` activates them for supported `auto`, required, and named tool requests. `auto` preserves the legacy conditional behavior: required or named `tool_choice`, or automatic tool choice when any tool has `strict=true` or `parallel_tool_calls` is false. `tool_choice=none` is unaffected by either setting. Only meaningful when `--dyn-enable-structural-tag` is set. + Controls when structural tags are activated. `always` activates them for supported `auto`, required, and named tool requests. `auto` preserves the legacy conditional behavior: required or named `tool_choice`, or automatic tool choice when any tool has `strict=true` or `parallel_tool_calls` is false. `tool_choice=none` is unaffected by either setting. Applies when structural tags are active. Allowed values: auto always @@ -277,7 +277,7 @@ Unless a field is marked environment-only, it has both a CLI flag and an environ - Controls parameter schema strictness inside structural tags. `auto` applies the declared parameter schema unless a tool explicitly sets `strict=false`; an omitted `strict` field is schema-enforced. `strict` applies the declared schema to every tool and overrides `strict=false`. Only meaningful when `--dyn-enable-structural-tag` is set. + Controls parameter schema strictness inside structural tags. `auto` applies the declared parameter schema unless a tool explicitly sets `strict=false`; an omitted `strict` field is schema-enforced. `strict` applies the declared schema to every tool and overrides `strict=false`. Applies when structural tags are active. Allowed values: auto strict diff --git a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md index a1bc6d343437..d030abac9687 100644 --- a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md +++ b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md @@ -1,5 +1,5 @@ --- -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: Structural Tag (Guided Decoding for Tool Calls) subtitle: Constrain model output to valid tool call format using xgrammar structural tags @@ -80,17 +80,18 @@ decoding. See [Activation Scope](#activation-scope) for the exact policy. | Flag | Values | Default | Description | |---|---|---|---| -| `--dyn-enable-structural-tag` | bool | `true` | Master switch for the Rust, Python vLLM, and Python SGLang frontend preprocessing paths. Explicitly disabling it prevents tool-call structural-tag injection. | +| `--dyn-enable-structural-tag` | bool | `true` | Enables tool-call structural tags in Rust, Python vLLM, and Python SGLang preprocessing. Rust retains native forced-tool tags for Kimi K2 required/named and Kimi K3 named choices when disabled. | | `--dyn-structural-tag-scope` | `auto`, `always` | `always` | Controls when structural tags are activated (see [Activation Scope](#activation-scope)). | | `--dyn-structural-tag-schema` | `auto`, `strict` | `auto` | Controls parameter schema strictness inside structural tags (see [Schema Modes](#schema-modes)). | ## Preserve Previous Behavior -To disable structural tags, pass the negated flag to the worker: +To disable optional structural guidance, pass the negated flag to the worker: ```bash python3 -m dynamo.vllm ... --no-dyn-enable-structural-tag python3 -m dynamo.sglang ... --no-dyn-enable-structural-tag +python3 -m dynamo.trtllm ... --no-dyn-enable-structural-tag ``` Or set the equivalent environment variable before starting the worker: @@ -99,6 +100,12 @@ Or set the equivalent environment variable before starting the worker: export DYN_ENABLE_STRUCTURAL_TAG=false ``` +Rust preprocessing preserves the existing native grammars for Kimi K2 +`required`/named choices and Kimi K3 named choices even when structural tags +are disabled. Kimi K2/K3 automatic choices and Kimi K3 `required` do not use +this exception. Python vLLM and SGLang preprocessing respect the opt-out for +all tool choices. + To keep structural tags enabled but preserve the previous conditional activation policy, set the scope to `auto`: @@ -110,10 +117,11 @@ With this scope, required and named tool choices remain eligible. Automatic tool choice uses structural tags only when a tool sets `strict: true` or the request sets `parallel_tool_calls` to `false`. -Setting `strict: false` on a tool relaxes its argument schema. On pinned vLLM -0.29.0, auto choice with every tool explicitly non-strict also gets no -structural tag from the registry; see [Schema Modes](#schema-modes). Use the -global opt-out above to disable structural tags for all requests. +Setting `strict: false` on a tool relaxes its argument schema. On vLLM +0.30.0, pinned by Dynamo's CUDA image, auto choice with every tool explicitly +non-strict also gets no structural tag from the registry; see +[Schema Modes](#schema-modes). Use the global opt-out above to disable optional +structural guidance. ## Supported Parsers @@ -138,9 +146,10 @@ therefore differ by backend and engine version. > [!NOTE] > Native Rust sidecars retain their existing conservative `off`/`auto` -> settings and are not included in this default change. The default-on policy -> applies when regular vLLM or SGLang workers publish the deployment runtime -> configuration consumed by frontend preprocessing. +> settings, including the native Kimi forced-tool exception described above, +> and are not included in this default change. The default-on policy +> applies when regular vLLM, SGLang, or TensorRT-LLM workers publish the +> deployment runtime configuration consumed by frontend preprocessing. ## Activation Scope @@ -185,7 +194,7 @@ the strongest safe tool envelope and relaxes that argument section. If the builder cannot produce a structural tag at all, Dynamo uses the existing compatibility path rather than introducing a new request error; automatic tool choice may therefore remain unconstrained for that parser/schema combination. -In the vLLM 0.29.0 version pinned by Dynamo, `tool_choice="auto"` returns no +In vLLM 0.30.0, pinned by Dynamo's CUDA image, `tool_choice="auto"` returns no structural tag when every tool is explicitly `strict: false`. The `always` activation scope still attempts the tag, but cannot enforce the native tool envelope for that request. Set schema mode to `strict` to override the opt-out diff --git a/lib/llm/src/http/service.rs b/lib/llm/src/http/service.rs index f7c368c7392c..1158d1b2e391 100644 --- a/lib/llm/src/http/service.rs +++ b/lib/llm/src/http/service.rs @@ -158,7 +158,6 @@ mod tests { fn kimi_k3_named_dynamic_tool_uses_structural_tag_policy() { let parsing_options = ParsingOptions { tool_call_parser: Some("kimi_k3".to_string()), - structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::On, ..Default::default() }; let result = apply_request_tool_call_parsing_options( @@ -212,7 +211,6 @@ mod tests { fn kimi_k2_required_resolves_to_the_real_structural_tag_decision() { let parsing_options = ParsingOptions { tool_call_parser: Some("kimi_k2".to_string()), - structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::On, ..Default::default() }; let result = @@ -221,7 +219,7 @@ mod tests { assert_eq!( result.guided_tool_constraint, GuidedToolConstraint::StructuralTag, - "kimi_k2 + required must use the enabled structural tag, not a reconstructed JSON schema" + "kimi_k2 + required must retain its native structural tag when mode is off" ); } @@ -229,7 +227,6 @@ mod tests { fn kimi_k2_named_resolves_to_the_real_structural_tag_decision() { let parsing_options = ParsingOptions { tool_call_parser: Some("kimi_k2".to_string()), - structural_tag_mode: crate::local_model::runtime_config::StructuralTagMode::On, ..Default::default() }; let named = json!({"type": "function", "function": {"name": "get_weather"}}); @@ -238,7 +235,7 @@ mod tests { assert_eq!( result.guided_tool_constraint, GuidedToolConstraint::StructuralTag, - "kimi_k2 + a named tool choice must use the enabled structural tag, not a reconstructed JSON schema" + "kimi_k2 + a named tool choice must retain its native structural tag when mode is off" ); } diff --git a/lib/llm/src/preprocessor/structural_tag.rs b/lib/llm/src/preprocessor/structural_tag.rs index c257efb5ca73..0f4c9735a55f 100644 --- a/lib/llm/src/preprocessor/structural_tag.rs +++ b/lib/llm/src/preprocessor/structural_tag.rs @@ -45,13 +45,21 @@ fn is_kimi_k2_parser(parser_name: Option<&str>) -> bool { parser_name == Some("kimi_k2") } +fn requires_intrinsic_structural_tag(parser_name: Option<&str>, tool_choice: &ToolChoice) -> bool { + // Preserve native forced-tool grammars even when optional guidance is off. + // K3 required retains its existing prompt/parser path. + (is_kimi_k2_parser(parser_name) + && matches!(tool_choice, ToolChoice::Required | ToolChoice::Named(_))) + || (is_kimi_k3_parser(parser_name) && matches!(tool_choice, ToolChoice::Named(_))) +} + pub(super) fn requires_native_tool_call_format( parser_name: Option<&str>, tool_choice: &ToolChoice, ) -> bool { // The generic forced-tool JSON grammar describes a different wire format - // from Kimi's native marker-delimited calls. This is fallback metadata, not - // an activation bypass: the deployment-level off switch remains absolute. + // from Kimi's native marker-delimited calls. This also covers K3 required, + // which uses its prompt/parser path when structural guidance is off. (is_kimi_k2_parser(parser_name) && matches!(tool_choice, ToolChoice::Required | ToolChoice::Named(_))) || (is_kimi_k3_parser(parser_name) @@ -67,7 +75,7 @@ fn should_skip_tool_call_ban(exclude_tools_when_none: bool, tool_choice: &ToolCh /// /// This is the single owner of "is a structural tag applicable and available" — /// covering the operator's global `structural_tag_mode`/`structural_tag_scope`, -/// the `tool_choice=None` ban-tag exclusion, and real +/// intrinsic Kimi forced-tool grammars, the `tool_choice=None` ban-tag exclusion, and real /// parser-registry builder availability. A caller can only /// reach `Required` by way of a real, registered /// [`dynamo_parsers::tool_calling::StructuralTagBuilder`] — it cannot ask for the @@ -111,7 +119,9 @@ pub(crate) fn structural_tag_decision( // prompt-level XTML path, but they still need an actual tool to require. validate_forced_tool_choice(tool_choice, tools)?; - if mode == StructuralTagMode::Off { + if mode == StructuralTagMode::Off + && !requires_intrinsic_structural_tag(parser_name, tool_choice) + { return Ok(StructuralTagDecision::NotApplicable); } @@ -452,13 +462,9 @@ mod tests { } #[test] - fn global_mode_off_is_authoritative_for_kimi_k2_required() { + fn kimi_forced_choices_keep_native_tags_when_global_mode_is_off() { let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); - let mut mdc = ModelDeploymentCard::load_from_disk(model_path, None).unwrap(); - mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; - mdc.runtime_config.tool_call_parser = Some("kimi_k2".to_string()); - let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); let tools = [ToolDefinition { name: "get_weather".to_string(), parameters: Some(serde_json::json!({ @@ -468,20 +474,48 @@ mod tests { })), strict: None, }]; - let mut request = preprocessed_request(); - - let applied = preprocessor - .apply_tool_choice_structural_tag( - &ToolChoice::Required, - &tools, - None, - false, - &mut request, - ) - .unwrap(); + for (parser, choice, marker) in [ + ( + "kimi_k2", + ToolChoice::Required, + "<|tool_calls_section_begin|>", + ), + ( + "kimi_k2", + ToolChoice::Named("get_weather".to_string()), + "<|tool_calls_section_begin|>", + ), + ( + "kimi_k3", + ToolChoice::Named("get_weather".to_string()), + "<|open|>call", + ), + ( + "kimi-k3", + ToolChoice::Named("get_weather".to_string()), + "<|open|>call", + ), + ] { + let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); + mdc.runtime_config.structural_tag_mode = StructuralTagMode::Off; + mdc.runtime_config.tool_call_parser = Some(parser.to_string()); + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let mut request = preprocessed_request(); + + let applied = preprocessor + .apply_tool_choice_structural_tag(&choice, &tools, None, false, &mut request) + .unwrap(); - assert!(!applied); - assert!(request.sampling_options.guided_decoding.is_none()); + assert!(applied, "{parser} + {choice:?} must retain its native tag"); + let tag = request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.structural_tag.as_ref()) + .expect("forced Kimi choice must install a native structural tag"); + assert!(tag.to_string().contains(marker)); + assert!(tag.to_string().contains("get_weather")); + } } #[test] diff --git a/lib/llm/src/preprocessor/tool_choice.rs b/lib/llm/src/preprocessor/tool_choice.rs index 99b5b407168b..403d488dac73 100644 --- a/lib/llm/src/preprocessor/tool_choice.rs +++ b/lib/llm/src/preprocessor/tool_choice.rs @@ -714,7 +714,7 @@ mod tests { } #[test] - fn structural_tag_off_keeps_kimi_k2_required_on_native_fallback() { + fn structural_tag_off_preserves_kimi_k2_required_native_tag() { let request = request(json!({ "tools": tools(), "tool_choice": "required" @@ -726,12 +726,19 @@ mod tests { .apply_tool_choice_guided_decoding(&request, &mut common_request, false) .unwrap(); - assert_eq!(applied, GuidedToolConstraint::None); - assert!(common_request.sampling_options.guided_decoding.is_none()); + assert_eq!(applied, GuidedToolConstraint::StructuralTag); + assert!( + common_request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.structural_tag.as_ref()) + .is_some() + ); } #[test] - fn structural_tag_off_keeps_kimi_k3_named_on_native_fallback() { + fn structural_tag_off_preserves_kimi_k3_named_native_tag() { let request = request(json!({ "tools": tools(), "tool_choice": { @@ -746,8 +753,15 @@ mod tests { .apply_tool_choice_guided_decoding(&request, &mut common_request, false) .unwrap(); - assert_eq!(applied, GuidedToolConstraint::None); - assert!(common_request.sampling_options.guided_decoding.is_none()); + assert_eq!(applied, GuidedToolConstraint::StructuralTag); + assert!( + common_request + .sampling_options + .guided_decoding + .as_ref() + .and_then(|guided| guided.structural_tag.as_ref()) + .is_some() + ); } #[test] @@ -774,6 +788,61 @@ mod tests { ); } + #[test] + fn kimi_k3_builder_error_uses_native_fallback() { + let request = request(json!({ + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "strict": true, + "parameters": { + "type": "object", + "properties": { + "value": { + "type": "string", + "$id": "urn:example:scoped", + "$defs": { + "allowed": {"type": "string", "enum": ["safe"]} + }, + "allOf": [{"$ref": "#/$defs/allowed"}] + } + }, + "required": ["value"] + } + } + }], + "tool_choice": "auto" + })); + let preprocessor = preprocessor("kimi_k3", StructuralTagMode::On); + let tools = convert_tools(request.inner.tools.as_deref().unwrap()); + let ctx = dynamo_parsers::tool_calling::ToolCallFormatBuildContext { + tool_choice: &ToolChoice::Auto, + tools: &tools, + parallel_tool_calls: request.inner.parallel_tool_calls, + schema_mode: preprocessor.runtime_config.structural_tag_schema, + starts_in_reasoning: false, + }; + // A scoped string reference cannot be safely rendered as raw XTML. + // Verify an actual builder error, rather than a policy skip or Ok(None). + let error = dynamo_parsers::tool_calling::StructuralTagBuilder::KimiK3 + .build_tool_call_format(&ctx) + .expect_err("scoped string reference must reach the builder-error fallback"); + let message = format!("{error:#}"); + assert!( + message.contains("value") && message.contains("string"), + "{message}" + ); + + let mut common_request = preprocessed_request(); + let applied = preprocessor + .apply_tool_choice_guided_decoding(&request, &mut common_request, false) + .expect("builder errors must allow the request to continue without a structural tag"); + + assert_eq!(applied, GuidedToolConstraint::None); + assert!(common_request.sampling_options.guided_decoding.is_none()); + } + #[test] fn kimi_k3_forced_choices_accept_dynamic_system_tools() { for tool_choice in [ From 5195452912bc96da5404eefee563af712ecf4d44 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Thu, 1 Oct 2026 00:43:38 -0700 Subject: [PATCH 07/15] Fix CI Signed-off-by: Indrajit Bhosale --- components/src/dynamo/frontend/prepost.py | 4 +- .../src/dynamo/frontend/sglang_prepost.py | 7 ++ ...est_pinned_vllm_structural_tag_registry.py | 10 ++- .../tests/test_sglang_processor_unit.py | 68 ++++++++++++++++++- .../vllm/tests/test_vllm_engine_generate.py | 9 ++- tests/report_pytest_markers.py | 4 ++ 6 files changed, 97 insertions(+), 5 deletions(-) diff --git a/components/src/dynamo/frontend/prepost.py b/components/src/dynamo/frontend/prepost.py index 24778e44ffab..83803e97eaa3 100644 --- a/components/src/dynamo/frontend/prepost.py +++ b/components/src/dynamo/frontend/prepost.py @@ -42,9 +42,11 @@ get_model_structural_tag: Any try: - from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag + from vllm.tool_parsers import structural_tag_registry except ImportError: # Older supported vLLM releases do not expose this registry. get_model_structural_tag = None +else: + get_model_structural_tag = structural_tag_registry.get_model_structural_tag if TYPE_CHECKING: from vllm.config import ModelConfig diff --git a/components/src/dynamo/frontend/sglang_prepost.py b/components/src/dynamo/frontend/sglang_prepost.py index c42ada78faa8..7cd5056e9bc6 100644 --- a/components/src/dynamo/frontend/sglang_prepost.py +++ b/components/src/dynamo/frontend/sglang_prepost.py @@ -679,6 +679,13 @@ def build_tool_call_guided_decoding( raw_tools, structural_tag_schema=structural_tag_schema, ) + if is_named_choice: + # SGLang's legacy tag builder ignores the name in tool_choice. + guidance_tools = [ + tool + for tool in guidance_tools or [] + if tool.function.name == sglang_tool_choice.function.name + ] tool_call_parser_name = _normalize_sglang_parser_name(tool_call_parser_name) try: parser = FunctionCallParser( diff --git a/components/src/dynamo/frontend/tests/test_pinned_vllm_structural_tag_registry.py b/components/src/dynamo/frontend/tests/test_pinned_vllm_structural_tag_registry.py index 6ccb71102fdb..86897daddd2d 100644 --- a/components/src/dynamo/frontend/tests/test_pinned_vllm_structural_tag_registry.py +++ b/components/src/dynamo/frontend/tests/test_pinned_vllm_structural_tag_registry.py @@ -4,8 +4,14 @@ """Pin the real vLLM structural-tag behavior used by Dynamo's auto tool path.""" import pytest -from vllm.entrypoints.openai.chat_completion.protocol import ChatCompletionToolsParam -from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag + +try: + from vllm.entrypoints.openai.chat_completion.protocol import ( + ChatCompletionToolsParam, + ) + from vllm.tool_parsers.structural_tag_registry import get_model_structural_tag +except ImportError: + pytest.skip("requires the vLLM structural-tag registry", allow_module_level=True) pytestmark = [ pytest.mark.unit, diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index 80dfded40ae0..8193e197faff 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -30,7 +30,10 @@ tool_choice_value, ) from sglang.srt.constrained.reasoner_grammar_backend import ReasonerGrammarObject -from sglang.srt.constrained.xgrammar_backend import XGrammarGrammar +from sglang.srt.constrained.xgrammar_backend import ( + XGrammarGrammar, + XGrammarGrammarBackend, +) from sglang.srt.function_call.function_call_parser import FunctionCallParser from sglang.srt.function_call.json_array_parser import JsonArrayParser from sglang.srt.utils.hf_transformers_utils import get_tokenizer @@ -102,6 +105,13 @@ def tool_grammar_compiler(tokenizer): ) +@pytest.fixture(scope="module") +def tool_grammar_backend(tokenizer): + backend = XGrammarGrammarBackend(tokenizer, vocab_size=len(tokenizer)) + yield backend + backend.executor.shutdown(wait=True) + + @pytest.fixture(scope="module") def byte_fallback_tokenizer(): return get_tokenizer(BYTE_FALLBACK_MODEL) @@ -1673,6 +1683,62 @@ def grammar_after_reasoning(): grammar.accept_token(token) +@pytest.mark.core +@pytest.mark.timeout(60) +@pytest.mark.filterwarnings( + "ignore:max_rollback_tokens is deprecated.*:DeprecationWarning" +) +@pytest.mark.parametrize("parser_name", ["qwen25", "qwen3_coder"]) +@pytest.mark.parametrize("tool_choice", ["auto", "required", "named"]) +def test_structural_tag_respects_named_tool_choice( + tool_grammar_backend, parser_name, tool_choice +): + """A named choice excludes other declared tools, including legacy tags.""" + raw_tools = [parity_tool(), parity_tool()] + raw_tools[1]["function"]["name"] = "get_forecast" + request = { + "tools": raw_tools, + "tool_choice": ( + {"type": "function", "function": {"name": "get_forecast"}} + if tool_choice == "named" + else tool_choice + ), + } + guided = build_tool_call_guided_decoding( + request, + tool_call_parser_name=parser_name, + sglang_tools=convert_tools(raw_tools), + structural_tag_mode="on", + structural_tag_scope="always", + ) + assert guided is not None + grammar = tool_grammar_backend.dispatch_structural_tag( + json.dumps(guided["structural_tag"]) + ) + assert isinstance(grammar, XGrammarGrammar) + + def accepts_call(name): + if parser_name == "qwen25": + call = ( + '\n{"name":"' + name + '", "arguments":' + '{"city":"Paris"}}\n' + ) + else: + call = ( + f"\n\n" + "\nParis\n\n" + "\n" + ) + return grammar.copy().matcher.accept_string(call) + + assert accepts_call("get_forecast") + assert accepts_call("get_weather") is (tool_choice != "named") + assert [tool["function"]["name"] for tool in raw_tools] == [ + "get_weather", + "get_forecast", + ] + + class TestBuildToolCallGuidedDecoding: # FRONTEND.3 — guided-decoding setup for tool_choice # Keep SGLang's guidance decisions aligned with the shared backend matrix. @pytest.mark.parametrize( diff --git a/components/src/dynamo/vllm/tests/test_vllm_engine_generate.py b/components/src/dynamo/vllm/tests/test_vllm_engine_generate.py index 0c383fd090a5..9a11446fd73b 100644 --- a/components/src/dynamo/vllm/tests/test_vllm_engine_generate.py +++ b/components/src/dynamo/vllm/tests/test_vllm_engine_generate.py @@ -316,7 +316,14 @@ def test_tito_adapter_rejects_unsupported_execution_paths( def test_tito_adapter_rejects_asymmetric_image_feature_objects(features): from dynamo.vllm.engine_generate import adapt_engine_generate_request - with pytest.raises(TypeError, match="hashes and placeholders must be lists"): + # vLLM 0.30 validates modality keys before Dynamo's image-list checks. + with pytest.raises( + (TypeError, ValueError), + match=( + "hashes and placeholders must be lists" + "|mm_hashes and mm_placeholders must use the same modalities" + ), + ): adapt_engine_generate_request( _request(features=features, sampling_params={"max_tokens": 1}), enable_multimodal=True, diff --git a/tests/report_pytest_markers.py b/tests/report_pytest_markers.py index d955bbb6b0ae..634a0a3f3d2c 100755 --- a/tests/report_pytest_markers.py +++ b/tests/report_pytest_markers.py @@ -161,6 +161,9 @@ "fsspec.implementations.dirfs", "sglang", "sglang.srt", + "sglang.srt.constrained", + "sglang.srt.constrained.reasoner_grammar_backend", + "sglang.srt.constrained.xgrammar_backend", "sglang.srt.entrypoints", "sglang.srt.entrypoints.openai", "sglang.srt.entrypoints.openai.protocol", @@ -187,6 +190,7 @@ "sglang.srt.disaggregation.utils", "sglang.srt.server_args", "sglang.srt.server_args_config_parser", + "xgrammar", "vllm", "vllm.config", "vllm.distributed", From 8499f07efffe02f63fe00a55d08166cbb0ad42d5 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Fri, 2 Oct 2026 16:02:10 -0700 Subject: [PATCH 08/15] Fix random infinite token generation in sglang Signed-off-by: Indrajit Bhosale --- components/src/dynamo/sglang/register.py | 45 +++++++++ .../sglang/tests/test_runtime_metadata.py | 71 ++++++++++++++ lib/llm/src/local_model/runtime_config.rs | 5 + lib/llm/src/preprocessor/structural_tag.rs | 94 +++++++++++++++++-- 4 files changed, 207 insertions(+), 8 deletions(-) diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index dcaf9e38a4ae..6aca00ec9ae0 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -8,6 +8,7 @@ from typing import Any, List, Optional import sglang as sgl +from sglang.srt.parser.reasoning_parser import ReasoningParser from sglang.srt.server_args import ServerArgs from sglang.srt.speculative.spec_info import SpeculativeAlgorithm @@ -31,6 +32,7 @@ register_model, ) from dynamo.sglang._compat import ( + filter_supported_async_generate_kwargs, sglang_uses_mla_backend, supports_disagg_prefill_cancel_anytime, ) @@ -56,6 +58,44 @@ SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake" SPEC_DECODE_RUNTIME_KEY = "spec_decode" +TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY = ( + "tool_call_structural_tag_reasoning_gate" +) + + +def publish_sglang_structural_tag_reasoning_policy( + runtime_config: ModelRuntimeConfig, + engine: Optional[sgl.Engine], + server_args: ServerArgs, +) -> None: + """Advertise the native gate that consumes reasoning before guided output.""" + # vLLM also publishes grammar timing through runtime metadata. + # SGLang's exclusion additionally depends on require_reasoning per request. + parser = getattr(server_args, "reasoning_parser", None) + if ( + engine is None + or not parser + or getattr(server_args, "skip_tokenizer_init", False) + ): + return + tokenizer = getattr(engine.tokenizer_manager, "tokenizer", None) + if ( + tokenizer is None + or "require_reasoning" + not in filter_supported_async_generate_kwargs( + engine, {"require_reasoning": True} + ) + ): + return + # Match the scheduler's gate initialization, including disabled gates when + # the reasoning terminator cannot be encoded by the initialized tokenizer. + reasoner = ReasoningParser( + model_type=parser, stream_reasoning=False, tokenizer=tokenizer + ) + if tokenizer.encode(reasoner.detector.think_end_token, add_special_tokens=False): + runtime_config.set_engine_specific( + TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, json.dumps(True) + ) def _supports_engine_generate( @@ -436,6 +476,11 @@ async def get_runtime_config( # set reasoning parser and tool call parser runtime_config.reasoning_parser = dynamo_args.dyn_reasoning_parser runtime_config.tool_call_parser = dynamo_args.dyn_tool_call_parser + # Multimodal and diffusion handlers do not forward the per-request gate. + if llm_handler and not getattr(server_args, "dllm_algorithm", None): + publish_sglang_structural_tag_reasoning_policy( + runtime_config, engine, server_args + ) if dynamo_args.dyn_default_thinking_mode is not None: runtime_config.set_engine_specific( "default_thinking_mode", diff --git a/components/src/dynamo/sglang/tests/test_runtime_metadata.py b/components/src/dynamo/sglang/tests/test_runtime_metadata.py index 0d1b94cbc181..110636cb4767 100644 --- a/components/src/dynamo/sglang/tests/test_runtime_metadata.py +++ b/components/src/dynamo/sglang/tests/test_runtime_metadata.py @@ -4,6 +4,7 @@ import asyncio import json from types import SimpleNamespace +from unittest.mock import Mock import pytest @@ -23,6 +24,62 @@ ] +@pytest.mark.parametrize( + "parser, skip_tokenizer_init, has_engine, has_tokenizer, supports_gate, think_end_ids, expected", + [ + ("kimi_k3", False, True, True, True, [163588, 39964, 163589], True), + (None, False, True, True, True, [163588], False), + ("kimi_k3", True, True, True, True, [163588], False), + ("kimi_k3", False, False, True, True, [163588], False), + ("kimi_k3", False, True, False, True, [163588], False), + ("kimi_k3", False, True, True, False, [163588], False), + ("kimi_k3", False, True, True, True, [], False), + ], +) +def test_structural_tag_reasoning_policy_advertises_only_an_available_gate( + parser, + skip_tokenizer_init, + has_engine, + has_tokenizer, + supports_gate, + think_end_ids, + expected, +): + from dynamo.sglang import register + + async def async_generate(*, require_reasoning=False): + pass + + async def legacy_async_generate(*, sampling_params=None): + pass + + tokenizer = Mock() + tokenizer.encode.return_value = think_end_ids + engine = ( + SimpleNamespace( + tokenizer_manager=SimpleNamespace( + tokenizer=tokenizer if has_tokenizer else None + ), + async_generate=async_generate if supports_gate else legacy_async_generate, + ) + if has_engine + else None + ) + server_args = SimpleNamespace( + reasoning_parser=parser, skip_tokenizer_init=skip_tokenizer_init + ) + runtime_config = register.ModelRuntimeConfig() + + register.publish_sglang_structural_tag_reasoning_policy( + runtime_config, engine, server_args + ) + + key = register.TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY + assert (key in runtime_config.runtime_data) is expected + if expected: + assert json.loads(runtime_config.runtime_data[key]) is True + + @pytest.mark.parametrize( "server_args, expected", [ @@ -162,6 +219,8 @@ def test_runtime_config_publishes_supported_disagg_capabilities( disaggregation_mode=disaggregation_mode, max_prefill_tokens=None, page_size=16, + reasoning_parser="kimi_k3", + skip_tokenizer_init=False, speculative_algorithm="NONE", speculative_num_steps=None, ) @@ -188,8 +247,16 @@ def test_runtime_config_publishes_supported_disagg_capabilities( ) engine = None if runtime_supported: + + async def async_generate(*, require_reasoning=False): + pass + + tokenizer = Mock() + tokenizer.encode.return_value = [163588, 39964, 163589] engine = SimpleNamespace( + async_generate=async_generate, tokenizer_manager=SimpleNamespace( + tokenizer=tokenizer, context_len=4096, validate_total_tokens=True, num_reserved_tokens=0, @@ -209,6 +276,10 @@ def test_runtime_config_publishes_supported_disagg_capabilities( assert (capability in runtime_config.runtime_data) is capability_expected if capability_expected: assert json.loads(runtime_config.runtime_data[capability]) is True + reasoning_gate = register.TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY + assert (reasoning_gate in runtime_config.runtime_data) is ( + runtime_supported and not enable_multimodal + ) assert "Failed to get runtime config" not in caplog.text diff --git a/lib/llm/src/local_model/runtime_config.rs b/lib/llm/src/local_model/runtime_config.rs index accf831b1409..181364f3838e 100644 --- a/lib/llm/src/local_model/runtime_config.rs +++ b/lib/llm/src/local_model/runtime_config.rs @@ -42,6 +42,11 @@ pub const TOKEN_BUDGET_RUNTIME_KEY: &str = "token_budget"; pub const TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY: &str = "tool_call_structural_tag_excludes_reasoning"; +/// A backend consumes the reasoning prefix only when `require_reasoning` is true +/// on the request. Missing metadata leaves reasoning in the frontend grammar. +pub const TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY: &str = + "tool_call_structural_tag_reasoning_gate"; + /// Describes which request-token overflows the frontend may reject early. /// /// The combined limit already accounts for engine-reserved tokens. A false diff --git a/lib/llm/src/preprocessor/structural_tag.rs b/lib/llm/src/preprocessor/structural_tag.rs index 0f4c9735a55f..45bca09450c6 100644 --- a/lib/llm/src/preprocessor/structural_tag.rs +++ b/lib/llm/src/preprocessor/structural_tag.rs @@ -5,6 +5,7 @@ use crate::local_model::runtime_config::{ StructuralTagMode, StructuralTagScope, TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, }; use crate::preprocessor::{OpenAIPreprocessor, PreprocessedRequest}; use crate::protocols::openai::tools::{ToolChoiceValidation, validate_tool_choice_against_names}; @@ -133,7 +134,7 @@ pub(crate) fn structural_tag_decision( } let Some(parser_name) = parser_name else { - tracing::warn!( + tracing::debug!( "Structural tag is enabled but --dyn-tool-call-parser is not set; \ structural tags will not be applied" ); @@ -203,23 +204,34 @@ impl OpenAIPreprocessor { parallel_tool_calls, schema_mode: self.runtime_config.structural_tag_schema, starts_in_reasoning: prompt_injected_reasoning - && !self.tool_call_structural_tag_excludes_reasoning(), + && !self.tool_call_structural_tag_excludes_reasoning( + preprocessed_request.require_reasoning, + ), }; Self::apply_tool_call_format(parser_name, builder, &ctx, preprocessed_request) } - fn tool_call_structural_tag_excludes_reasoning(&self) -> bool { - match self - .runtime_config - .get_engine_specific::(TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY) - { + fn tool_call_structural_tag_excludes_reasoning(&self, require_reasoning: bool) -> bool { + // vLLM advertises exclusion for the deployment; SGLang gates it per request. + // Once the backend consumes THINK_CLOSE, this grammar must not expect it again. + // Auto tool choice alone does not activate SGLang's gate. + self.structural_tag_reasoning_metadata( + TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + ) || (require_reasoning + && self.structural_tag_reasoning_metadata( + TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, + )) + } + + fn structural_tag_reasoning_metadata(&self, key: &str) -> bool { + match self.runtime_config.get_engine_specific::(key) { Ok(Some(excludes_reasoning)) => excludes_reasoning, Ok(None) => false, Err(error) => { tracing::warn!( %error, - key = TOOL_CALL_STRUCTURAL_TAG_EXCLUDES_REASONING_RUNTIME_KEY, + key, "Ignoring invalid structural-tag reasoning metadata; using the compatibility behavior" ); false @@ -433,6 +445,72 @@ mod tests { ); } + #[test] + fn kimi_k3_reasoning_gate_avoids_duplicate_prefix_per_request() { + let model_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("tests/data/sample-models/mock-llama-3.1-8b-instruct"); + let tools = [ToolDefinition { + name: "get_weather".to_string(), + parameters: Some(serde_json::json!({ + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"] + })), + strict: None, + }]; + for (gate, require_reasoning, prompt_injected_reasoning, choice, expects_prefix) in [ + (Some(true), true, true, ToolChoice::Required, false), + (None, true, true, ToolChoice::Required, true), + (Some(false), true, true, ToolChoice::Required, true), + (Some(true), false, true, ToolChoice::Auto, true), + (Some(true), true, false, ToolChoice::Required, false), + ( + Some(true), + false, + false, + ToolChoice::Named("get_weather".to_string()), + false, + ), + ] { + let mut mdc = ModelDeploymentCard::load_from_disk(&model_path, None).unwrap(); + mdc.runtime_config.structural_tag_mode = StructuralTagMode::On; + mdc.runtime_config.structural_tag_scope = StructuralTagScope::Always; + mdc.runtime_config.tool_call_parser = Some("kimi_k3".to_string()); + if let Some(gate) = gate { + mdc.runtime_config + .set_engine_specific(TOOL_CALL_STRUCTURAL_TAG_REASONING_GATE_RUNTIME_KEY, gate) + .unwrap(); + } + let preprocessor = OpenAIPreprocessor::new(mdc).unwrap(); + let mut request = preprocessed_request(); + request.require_reasoning = require_reasoning; + assert!( + preprocessor + .apply_tool_choice_structural_tag( + &choice, + &tools, + None, + prompt_injected_reasoning, + &mut request, + ) + .unwrap(), + "gate={gate:?}, require_reasoning={require_reasoning}, choice={choice:?}" + ); + let tag = request + .sampling_options + .guided_decoding + .unwrap() + .structural_tag + .unwrap(); + let has_prefix = tag["format"]["elements"][0]["end"] == "<|close|>think<|sep|>"; + assert_eq!( + has_prefix, expects_prefix, + "gate={gate:?}, require_reasoning={require_reasoning}, prompt_injected_reasoning={prompt_injected_reasoning}, choice={choice:?}" + ); + assert!(tag.to_string().contains("<|open|>tools<|sep|>")); + } + } + #[test] fn kimi_forced_choices_are_classified_as_native_format_fallbacks() { let named = ToolChoice::Named("get_weather".to_string()); From 5c8afd595e4ab2d55f4c2ea86faea5d87332f3db Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Fri, 2 Oct 2026 22:05:08 -0700 Subject: [PATCH 09/15] Update crates Signed-off-by: Indrajit Bhosale --- Cargo.lock | 5 +++-- Cargo.toml | 2 +- lib/bindings/kvbm/Cargo.lock | 5 +++-- lib/bindings/python/Cargo.lock | 5 +++-- lib/bindings/python/Cargo.toml | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e8c181cbc0b..2978528507ae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2875,9 +2875,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.0" +version = "9.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d4ac14f31d8af179e8c312d54529bbfa732c954fc79ded7e29feabf3824379d" +checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" dependencies = [ "aho-corasick", "anyhow", @@ -2886,6 +2886,7 @@ dependencies = [ "futures", "num-traits", "openai-harmony", + "percent-encoding", "regex", "rustpython-ruff_python_ast", "rustpython-ruff_python_parser", diff --git a/Cargo.toml b/Cargo.toml index 0c687cd11996..83cf8d3687d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,7 @@ dynamo-memory = { path = "lib/memory", version = "1.6.0" } dynamo-mocker = { path = "lib/mocker", version = "1.6.0" } dynamo-kv-router = { path = "lib/kv-router", version = "1.6.0" } dynamo-protocols = { version = "=6.1.0" } -dynamo-parsers = { version = "=9.2.0" } +dynamo-parsers = { version = "=9.2.6" } # Published on crates.io. To see all available versions: # web: https://crates.io/crates/dynamo-parsers-v2/versions diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index dd147124b7de..787aed8b4868 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -2295,9 +2295,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.0" +version = "9.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d4ac14f31d8af179e8c312d54529bbfa732c954fc79ded7e29feabf3824379d" +checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" dependencies = [ "aho-corasick", "anyhow", @@ -2306,6 +2306,7 @@ dependencies = [ "futures", "num-traits", "openai-harmony", + "percent-encoding", "regex", "rustpython-ruff_python_ast", "rustpython-ruff_python_parser", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index 2010c54084b0..f07643e47631 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2347,9 +2347,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.0" +version = "9.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d4ac14f31d8af179e8c312d54529bbfa732c954fc79ded7e29feabf3824379d" +checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" dependencies = [ "aho-corasick", "anyhow", @@ -2358,6 +2358,7 @@ dependencies = [ "futures", "num-traits", "openai-harmony", + "percent-encoding", "regex", "rustpython-ruff_python_ast", "rustpython-ruff_python_parser", diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index 8505c40b350d..29de4fd52a81 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -62,7 +62,7 @@ request-trace-s3 = ["dynamo-llm/request-trace-s3"] aisimulate-core = { version = "=0.13.0-dev.202609300000000061", optional = true, features = ["python"] } dynamo-runtime = { path = "../../runtime" } dynamo-mocker = { path = "../../mocker" } -dynamo-parsers = { version = "=9.2.0" } +dynamo-parsers = { version = "=9.2.6" } dynamo-backend-common = { path = "../../backend-common" } dynamo-sidecar-common = { path = "../../sidecar/common" } dynamo-sglang-sidecar = { path = "../../sidecar/sglang" } From 83216ca23e383363c8acb5c3da4b6a0d0e0dd6a0 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Fri, 2 Oct 2026 22:22:40 -0700 Subject: [PATCH 10/15] Update crates Signed-off-by: Indrajit Bhosale --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- lib/bindings/kvbm/Cargo.lock | 4 ++-- lib/bindings/python/Cargo.lock | 4 ++-- lib/bindings/python/Cargo.toml | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2978528507ae..e816a781ddf4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2875,9 +2875,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.6" +version = "9.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" +checksum = "a751a31502748c656f69bd1fab01f1d844b6bb629b69e9fbed0b0da9af916b7a" dependencies = [ "aho-corasick", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 83cf8d3687d4..ac50eb8923b6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,7 @@ dynamo-memory = { path = "lib/memory", version = "1.6.0" } dynamo-mocker = { path = "lib/mocker", version = "1.6.0" } dynamo-kv-router = { path = "lib/kv-router", version = "1.6.0" } dynamo-protocols = { version = "=6.1.0" } -dynamo-parsers = { version = "=9.2.6" } +dynamo-parsers = { version = "=9.2.7" } # Published on crates.io. To see all available versions: # web: https://crates.io/crates/dynamo-parsers-v2/versions diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index 787aed8b4868..6bcecfebf190 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -2295,9 +2295,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.6" +version = "9.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" +checksum = "a751a31502748c656f69bd1fab01f1d844b6bb629b69e9fbed0b0da9af916b7a" dependencies = [ "aho-corasick", "anyhow", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index f07643e47631..f9d71f5696e5 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2347,9 +2347,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "9.2.6" +version = "9.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedc2cc2376c35d3bc998f6ec326271703cd19a61ef18ad7e1d21dbde3eeb638" +checksum = "a751a31502748c656f69bd1fab01f1d844b6bb629b69e9fbed0b0da9af916b7a" dependencies = [ "aho-corasick", "anyhow", diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index 29de4fd52a81..7b1c3ba30017 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -62,7 +62,7 @@ request-trace-s3 = ["dynamo-llm/request-trace-s3"] aisimulate-core = { version = "=0.13.0-dev.202609300000000061", optional = true, features = ["python"] } dynamo-runtime = { path = "../../runtime" } dynamo-mocker = { path = "../../mocker" } -dynamo-parsers = { version = "=9.2.6" } +dynamo-parsers = { version = "=9.2.7" } dynamo-backend-common = { path = "../../backend-common" } dynamo-sidecar-common = { path = "../../sidecar/common" } dynamo-sglang-sidecar = { path = "../../sidecar/sglang" } From 4b1fa4d1444685eb23821c2b51bd429ec04325a6 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Fri, 2 Oct 2026 22:55:18 -0700 Subject: [PATCH 11/15] Fix fialing test Signed-off-by: Indrajit Bhosale --- .../dynamo/frontend/tests/test_sglang_checkpoint_templates.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/components/src/dynamo/frontend/tests/test_sglang_checkpoint_templates.py b/components/src/dynamo/frontend/tests/test_sglang_checkpoint_templates.py index 6f930eb58ea9..8c26eae7ac21 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_checkpoint_templates.py +++ b/components/src/dynamo/frontend/tests/test_sglang_checkpoint_templates.py @@ -143,5 +143,7 @@ def constraint(parser, tool_choice): tokenizer=HunyuanTokenizer(), tool_call_parser_name="hunyuan", reasoning_parser_name="hunyuan", + structural_tag_mode="on", + structural_tag_scope="always", ) assert result.guided_decoding == {"structural_tag": {"type": "object"}} From dd482e111a9a4b369c32fea29c19b39bcb7397b3 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Mon, 5 Oct 2026 13:01:06 -0700 Subject: [PATCH 12/15] Update comments Signed-off-by: Indrajit Bhosale --- .../dynamo/common/configuration/groups/runtime_args.py | 3 ++- .../tool-calling-and-reasoning/structural-tag.md | 10 ++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/components/src/dynamo/common/configuration/groups/runtime_args.py b/components/src/dynamo/common/configuration/groups/runtime_args.py index 14c967a2b012..30970ef35986 100644 --- a/components/src/dynamo/common/configuration/groups/runtime_args.py +++ b/components/src/dynamo/common/configuration/groups/runtime_args.py @@ -297,7 +297,8 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: help="Enable structural tag guided decoding for tool calls when the configured " "parser and backend support it. Disabling this flag suppresses optional " "guidance; Rust retains native tags for Kimi K2 required/named and " - "Kimi K3 named tool choices. Python vLLM and SGLang respect the opt-out.", + "Kimi K3 named tool choices. Python vLLM and SGLang respect the opt-out. " + "Unsupported schema constructs can cause backend grammar compilation errors.", ) add_argument( g, diff --git a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md index d030abac9687..4d30b7111f32 100644 --- a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md +++ b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md @@ -194,6 +194,16 @@ the strongest safe tool envelope and relaxes that argument section. If the builder cannot produce a structural tag at all, Dynamo uses the existing compatibility path rather than introducing a new request error; automatic tool choice may therefore remain unconstrained for that parser/schema combination. + +> [!WARNING] +> Valid tool schemas containing constructs unsupported by the backend's +> XGrammar version, such as regex lookahead, can cause request rejection. +> Backend compilation errors do not trigger Dynamo's builder fallback. +> With XGrammar 0.2.1 or 0.2.7 and schema mode `auto`, setting the tool +> function's `strict: false` avoids this lookahead rejection by disabling +> argument-schema enforcement. This workaround is not guaranteed on older +> versions. The deployment opt-out still retains Rust Kimi K3 named-call tags. + In vLLM 0.30.0, pinned by Dynamo's CUDA image, `tool_choice="auto"` returns no structural tag when every tool is explicitly `strict: false`. The `always` activation scope still attempts the tag, but cannot enforce the native tool From 2f5313af9ee35ad578d3011975eca03b46305185 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Mon, 5 Oct 2026 17:21:10 -0700 Subject: [PATCH 13/15] Update docs Signed-off-by: Indrajit Bhosale --- .../tool-calling-and-reasoning/structural-tag.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md index 4d30b7111f32..84c262c34703 100644 --- a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md +++ b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md @@ -100,11 +100,14 @@ Or set the equivalent environment variable before starting the worker: export DYN_ENABLE_STRUCTURAL_TAG=false ``` -Rust preprocessing preserves the existing native grammars for Kimi K2 -`required`/named choices and Kimi K3 named choices even when structural tags -are disabled. Kimi K2/K3 automatic choices and Kimi K3 `required` do not use -this exception. Python vLLM and SGLang preprocessing respect the opt-out for -all tool choices. +Rust preprocessing still applies native tags to Kimi K2 `required`/named +choices and Kimi K3 named choices when structural tags are disabled. Kimi K3 +named tags use the new argument-schema enforcement; the opt-out does not +restore main's grammar. A lookahead pattern can still be rejected by XGrammar +0.2.1 or 0.2.7 unless the tool sets `strict: false` under schema mode `auto`. +Global schema mode `strict` overrides that workaround. Kimi K2/K3 automatic +choices and Kimi K3 `required` do not use this exception. Python vLLM and SGLang +preprocessing respect the opt-out for all tool choices. To keep structural tags enabled but preserve the previous conditional activation policy, set the scope to `auto`: From e556cfaf8dcb51c5c9f71a5b265542995c93052f Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 7 Oct 2026 13:10:50 -0700 Subject: [PATCH 14/15] Update Docs Signed-off-by: Indrajit Bhosale --- .../tool-calling-and-reasoning/structural-tag.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md index 84c262c34703..a54b02bde23e 100644 --- a/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md +++ b/docs/fern/pages/use-cases/tool-calling-and-reasoning/structural-tag.md @@ -34,6 +34,14 @@ Benefits: Structural tags are enabled by default. Configure the tool-call parser on the **worker**; the Frontend needs no extra flags: +> [!NOTE] +> TensorRT-LLM guided decoding remains **opt-in** pending +> [TensorRT-LLM #19913](https://github.com/NVIDIA/TensorRT-LLM/issues/19913). +> Its `--guided-decoding-backend` default is unset, so default deployments ignore +> structural tags and do not enforce their tool argument schemas. For affected +> deployments using `--guided-decoding-backend xgrammar`, pass +> `--no-dyn-enable-structural-tag` until a supported upgrade resolves the issue. + ```yaml apiVersion: nvidia.com/v1beta1 kind: DynamoGraphDeployment From f17f39f412771161f8c6f1aa312428a8f9381db0 Mon Sep 17 00:00:00 2001 From: Indrajit Bhosale Date: Wed, 7 Oct 2026 15:43:21 -0700 Subject: [PATCH 15/15] Fix failing CI Signed-off-by: Indrajit Bhosale --- .../src/dynamo/frontend/tests/test_vllm_processor_unit.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index e31cee3b2e8d..0989298a8f92 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -1397,6 +1397,8 @@ class RequestForSampling(SimpleNamespace): max_tokens=1, logprobs=True, top_logprobs=1, + top_k=None, + min_p=None, cache_salt=None, mm_processor_kwargs=None, ), @@ -1662,6 +1664,7 @@ class RequestForSampling(SimpleNamespace): max_tokens=1, cache_salt=None, mm_processor_kwargs=None, + logprobs=None, **{"temperature": None, "top_k": None, "min_p": None, **requested}, ) monkeypatch.setattr(