diff --git a/rust/Cargo.lock b/rust/Cargo.lock index e580589b1e35..743633b447e7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -316,6 +316,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "auto_impl" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -5726,6 +5737,7 @@ dependencies = [ "vllm-text", "vllm-tokenizer", "vllm-tool-parser", + "xgrammar-structural-tag", "zeromq", ] @@ -5981,6 +5993,7 @@ dependencies = [ "thiserror-ext", "tool-parser", "winnow", + "xgrammar-structural-tag", ] [[package]] @@ -6582,6 +6595,19 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xgrammar-structural-tag" +version = "0.1.0+xgrammar.0.2.2.4d145cc" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2436dea2393d55a3b188588aa300c5a8afe8f45a77da52c611fb4498a6c876e6" +dependencies = [ + "auto_impl", + "serde", + "serde_json", + "strum", + "thiserror 2.0.18", +] + [[package]] name = "y4m" version = "0.8.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 455e660bcfee..e31bf07bbd47 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -123,6 +123,7 @@ vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } vllm-tool-parser = { path = "src/tool-parser" } winnow = "1.0.2" +xgrammar-structural-tag = "0.1.0" zeromq = { version = "0.6.0", default-features = false, features = [ "tokio-runtime", "all-transport", diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 0523b9defe93..85368ad98b92 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -34,6 +34,7 @@ vllm-reasoning-parser.workspace = true vllm-text.workspace = true vllm-tokenizer.workspace = true vllm-tool-parser.workspace = true +xgrammar-structural-tag.workspace = true [dev-dependencies] anyhow.workspace = true diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index bbd99572004d..c472a65601d5 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -64,6 +64,8 @@ pub enum Error { StreamClosedBeforeTerminalOutput { request_id: String }, #[error("tool call stream state is inconsistent: {message}")] ToolCallStreamInvariant { message: String }, + #[error("failed to build structural tag: {message}")] + StructuralTag { message: String }, #[error(transparent)] Text(#[from] vllm_text::Error), #[error(transparent)] diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index bebcf8839d52..dbc9cc05a517 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -1,6 +1,7 @@ //! Default output processing pipeline. mod reasoning; +mod structural_tag; mod tool; use std::sync::Once; @@ -11,6 +12,7 @@ use trait_set::trait_set; use vllm_text::tokenizer::DynTokenizer; use self::reasoning::reasoning_event_stream; +use self::structural_tag::apply_structural_tag_constraint; use self::tool::tool_event_stream; use super::structured::structured_chat_event_stream; use crate::error::Result; @@ -21,7 +23,7 @@ use crate::output::{ use crate::parser::ParserSelection; use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; -use crate::request::{ChatRequest, ChatToolChoice}; +use crate::request::ChatRequest; use crate::{Error, Result as ChatResult}; trait_set! { @@ -54,8 +56,7 @@ impl DefaultChatOutputProcessor { tool_call_parser: &ParserSelection, reasoning_parser: &ParserSelection, ) -> ChatResult { - let tool_parsing_enabled = - matches!(request.tool_choice, ChatToolChoice::Auto) && !request.tools.is_empty(); + let tool_parsing_enabled = request.tool_parsing_enabled(); let tool_parser = if tool_parsing_enabled { Some(Self::resolve_tool_parser( request, @@ -115,6 +116,8 @@ impl DefaultChatOutputProcessor { request.decode_options.skip_special_tokens = false; } + apply_structural_tag_constraint(request, parser.as_ref())?; + TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser")); Ok(parser) } diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs new file mode 100644 index 000000000000..eb1fa3d14362 --- /dev/null +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -0,0 +1,257 @@ +//! Applies xgrammar structural-tag constraints for strict tool calling. + +use thiserror_ext::AsReport; +use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams}; +use xgrammar_structural_tag::{ + FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam, + build_structural_tag, +}; + +use crate::parser::tool::ToolParser; +use crate::request::{ChatRequest, ChatToolChoice}; +use crate::{Error, Result as ChatResult}; + +/// Apply structural tag constraints to the request based on the tool parser's structural tag +/// support and the request's tool choice. +pub(super) fn apply_structural_tag_constraint( + request: &mut ChatRequest, + parser: &dyn ToolParser, +) -> ChatResult<()> { + let Some(model) = parser.structural_tag_model() else { + return Ok(()); + }; + let Some(tool_choice) = structural_tag_tool_choice(request) else { + return Ok(()); + }; + + let tools = request + .tools + .iter() + .map(|tool| { + ToolParam::Function(FunctionToolParam::new(FunctionDefinition { + name: tool.name.clone(), + description: tool.description.clone(), + parameters: Some(tool.parameters.clone()), + strict: tool.strict, + })) + }) + .collect::>(); + + let structural_tag = build_structural_tag(model, &tools, tool_choice, false) + .and_then(|tag| tag.to_json_string()) + .map_err(|error| Error::StructuralTag { + message: error.to_report_string(), + })?; + + // Overwrite any existing structured output settings with the structural tag constraint. + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + structural_tag: Some(structural_tag), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + + Ok(()) +} + +/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request. +/// +/// Returns `None` if no structural tag constraints should be applied. +fn structural_tag_tool_choice(request: &ChatRequest) -> Option { + if request.tools.is_empty() { + return None; + } + + match &request.tool_choice { + // For `Auto`, only apply the structural tag if there's at least one strict tool. + ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => { + Some(StructuralTagToolChoice::auto()) + } + ChatToolChoice::Auto | ChatToolChoice::None => None, + + ChatToolChoice::Required => Some(StructuralTagToolChoice::required()), + ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())), + } +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams}; + use vllm_tool_parser::{Qwen3CoderToolParser, Tool}; + + use super::*; + + fn chat_tool(name: &str, strict: Option) -> Tool { + Tool { + name: name.to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": { + "query": { "type": "string" } + }, + "required": ["query"] + }), + strict, + } + } + + fn qwen3_coder_parser(tools: &[Tool]) -> Box { + Qwen3CoderToolParser::create(tools).expect("Qwen3 Coder parser should build") + } + + fn request(tool_choice: ChatToolChoice, tools: Vec) -> ChatRequest { + ChatRequest { + tool_choice, + tools, + ..ChatRequest::for_test() + } + } + + fn structural_tag_value(request: &ChatRequest) -> Value { + let params = request + .sampling_params + .structured_outputs + .as_ref() + .expect("structured outputs should be set"); + assert_eq!(params.backend, StructuredOutputBackend::Xgrammar); + serde_json::from_str( + params.structural_tag.as_deref().expect("structural_tag should be set"), + ) + .expect("structural_tag should be valid JSON") + } + + fn structured_outputs(request: &ChatRequest) -> &StructuredOutputsParams { + request + .sampling_params + .structured_outputs + .as_ref() + .expect("structured outputs should be set") + } + + #[test] + fn auto_strict_tool_choice_builds_structural_tag() { + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn auto_non_strict_tool_choice_skips_structural_tag() { + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag decision should succeed"); + + assert!(request.sampling_params.structured_outputs.is_none()); + } + + #[test] + fn auto_strict_tool_choice_overwrites_existing_json_guidance() { + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + json: Some(json!({"type": "object"})), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag should build"); + + let params = structured_outputs(&request); + assert!(params.json.is_none()); + assert!(params.structural_tag.is_some()); + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn required_tool_choice_builds_structural_tag_without_strict_tools() { + let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn required_tool_choice_overwrites_existing_json_object_guidance() { + let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + json_object: Some(true), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag should build"); + + let params = structured_outputs(&request); + assert!(params.json_object.is_none()); + assert!(params.structural_tag.is_some()); + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn named_tool_choice_builds_structural_tag_for_named_tool_only() { + let mut request = request( + ChatToolChoice::Function { + name: "lookup".to_string(), + }, + vec![chat_tool("search", None), chat_tool("lookup", None)], + ); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request).to_string(); + assert!(tag.contains("lookup")); + assert!(!tag.contains("search")); + } + + #[test] + fn none_tool_choice_skips_structural_tag() { + let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag decision should succeed"); + + assert!(request.sampling_params.structured_outputs.is_none()); + } + + #[test] + fn none_tool_choice_preserves_existing_json_object_guidance() { + let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + json_object: Some(true), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag decision should succeed"); + + let params = structured_outputs(&request); + assert_eq!(params.json_object, Some(true)); + assert!(params.structural_tag.is_none()); + } +} diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index 7b9ae5f663e0..51b2efebd417 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -382,12 +382,16 @@ impl ChatOptions { } /// Tool-choice semantics supported by `vllm-chat`. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ChatToolChoice { - Auto, #[default] None, + Auto, + Required, + Function { + name: String, + }, } /// One chat request ready to be rendered into a prompt and lowered into a @@ -486,7 +490,7 @@ impl ChatRequest { /// Return true if this request should enable tool parsing based on the tool /// choice and tool list. pub(crate) fn tool_parsing_enabled(&self) -> bool { - matches!(self.tool_choice, ChatToolChoice::Auto) && !self.tools.is_empty() + !matches!(self.tool_choice, ChatToolChoice::None) && !self.tools.is_empty() } /// Return the request-level thinking toggle when explicitly requested. diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 0462294b0c29..2d64d194bf2e 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -361,6 +361,13 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result Ok(ChatToolChoice::Auto), Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None), + Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required), + Some(ToolChoice::Function { + tool_type, + function, + }) if tool_type == "function" => Ok(ChatToolChoice::Function { + name: function.name.clone(), + }), _ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice), } } @@ -962,6 +969,74 @@ mod tests { assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None); } + #[test] + fn prepare_chat_request_lowers_required_tool_choice() { + let request = ChatCompletionRequest { + tools: Some(vec![Tool { + tool_type: "function".to_string(), + function: Function { + name: "get_weather".to_string(), + description: Some("Get weather".to_string()), + parameters: json!({ + "type": "object", + "properties": {"city": {"type": "string"}}, + }), + strict: None, + }, + }]), + tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)), + ..base_request() + }; + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required); + } + + #[test] + fn prepare_chat_request_lowers_named_function_tool_choice() { + let request = ChatCompletionRequest { + tools: Some(vec![Tool { + tool_type: "function".to_string(), + function: Function { + name: "get_weather".to_string(), + description: Some("Get weather".to_string()), + parameters: json!({ + "type": "object", + "properties": {"city": {"type": "string"}}, + }), + strict: None, + }, + }]), + tool_choice: Some(ToolChoice::Function { + tool_type: "function".to_string(), + function: crate::routes::openai::utils::types::FunctionChoice { + name: "get_weather".to_string(), + }, + }), + ..base_request() + }; + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert_eq!( + prepared.chat_request.tool_choice, + ChatToolChoice::Function { + name: "get_weather".to_string(), + } + ); + } + #[test] fn prepare_chat_request_lowers_logprobs_fields() { let request = ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index 2e789573d219..f731cb2cde18 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -1,6 +1,6 @@ use super::types::ChatCompletionRequest; use crate::error::{ApiError, bail_invalid_request}; -use crate::routes::openai::utils::types::{ChatMessage, Tool, ToolChoice, ToolChoiceValue}; +use crate::routes::openai::utils::types::{ChatMessage, Tool}; /// Enforce the minimal compatibility contract for the Rust OpenAI server. pub(super) fn validate_request_compat( @@ -58,30 +58,6 @@ pub(super) fn validate_request_compat( } } - if let Some(tool_choice) = &request.tool_choice { - match tool_choice { - ToolChoice::Value(ToolChoiceValue::Auto | ToolChoiceValue::None) => {} - ToolChoice::Value(ToolChoiceValue::Required) => { - bail_invalid_request!( - param = "tool_choice", - "tool_choice=required is not supported yet." - ); - } - ToolChoice::Function { .. } => { - bail_invalid_request!( - param = "tool_choice", - "Named function tool_choice is not supported yet." - ); - } - ToolChoice::AllowedTools { .. } => { - bail_invalid_request!( - param = "tool_choice", - "allowed_tools tool_choice is not supported yet." - ); - } - } - } - if request.use_beam_search { bail_invalid_request!( param = "use_beam_search", @@ -159,8 +135,7 @@ mod tests { use crate::routes::openai::chat_completions::types::ChatCompletionRequest; use crate::routes::openai::utils::structured_outputs::ResponseFormat; use crate::routes::openai::utils::types::{ - ChatMessage, Function, FunctionChoice, MessageContent, StringOrArray, Tool, ToolChoice, - ToolChoiceValue, ToolReference, + ChatMessage, Function, MessageContent, StringOrArray, Tool, ToolChoice, ToolChoiceValue, }; fn served(names: &[&str]) -> Vec { @@ -357,38 +332,4 @@ mod tests { validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) .expect("tool_choice=none is ok"); } - - #[test] - fn validate_request_compat_rejects_required_and_named_tool_choices() { - let required = ChatCompletionRequest { - tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)), - ..base_request() - }; - assert!(validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); - - let named = ChatCompletionRequest { - tool_choice: Some(ToolChoice::Function { - tool_type: "function".to_string(), - function: FunctionChoice { - name: "tool".to_string(), - }, - }), - ..base_request() - }; - assert!(validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); - - let allowed_tools = ChatCompletionRequest { - tool_choice: Some(ToolChoice::AllowedTools { - tool_type: "allowed_tools".to_string(), - mode: "auto".to_string(), - tools: vec![ToolReference::Function { - name: "tool".to_string(), - }], - }), - ..base_request() - }; - assert!( - validate_request_compat(&allowed_tools, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err() - ); - } } diff --git a/rust/src/tool-parser/Cargo.toml b/rust/src/tool-parser/Cargo.toml index 0bc7010b75ba..c4363906aa05 100644 --- a/rust/src/tool-parser/Cargo.toml +++ b/rust/src/tool-parser/Cargo.toml @@ -14,6 +14,7 @@ serde_json.workspace = true thiserror.workspace = true thiserror-ext.workspace = true winnow.workspace = true +xgrammar-structural-tag.workspace = true [dev-dependencies] criterion.workspace = true diff --git a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs b/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs index abca33336a7b..bc636c6035a1 100644 --- a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs +++ b/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs @@ -1,5 +1,5 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3.2 models. /// @@ -44,6 +44,10 @@ impl ToolParser for DeepSeekV32ToolParser { true } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::DeepSeekV32) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs b/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs index ba01b1b586a3..9047b24ced70 100644 --- a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs +++ b/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs @@ -1,5 +1,5 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V4 models. /// @@ -47,6 +47,10 @@ impl ToolParser for DeepSeekV4ToolParser { true } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::DeepSeekV4) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } @@ -65,8 +69,8 @@ mod tests { use serde_json::{Value, json}; use super::DeepSeekV4ToolParser; - use crate::ToolParserTestExt as _; use crate::test_utils::{collect_stream, test_tools}; + use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { let params = params @@ -83,6 +87,16 @@ mod tests { ) } + #[test] + fn deepseek_v4_exposes_structural_tag_model() { + let parser = DeepSeekV4ToolParser::new(&test_tools()); + + assert_eq!( + parser.structural_tag_model(), + Some(StructuralTagModel::DeepSeekV4) + ); + } + #[test] fn deepseek_v4_parse_complete_reuses_dsml_parser_with_tool_calls_token() { let mut parser = DeepSeekV4ToolParser::new(&test_tools()); diff --git a/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs b/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs index 9c8a2a5c5850..6d6062432ab5 100644 --- a/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs +++ b/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs @@ -1,5 +1,5 @@ use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3 JSON-fenced tool calls. /// @@ -32,6 +32,10 @@ impl ToolParser for DeepSeekV3ToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::DeepSeekR1) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs b/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs index d16ffd7de6e7..33b362439a86 100644 --- a/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs +++ b/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs @@ -1,5 +1,5 @@ use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3.1 raw JSON tool calls. /// @@ -28,6 +28,10 @@ impl ToolParser for DeepSeekV31ToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::DeepSeekV31) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/glm_xml/glm47_moe.rs b/rust/src/tool-parser/src/glm_xml/glm47_moe.rs index 3d1c38d55f7a..74afd6c250b4 100644 --- a/rust/src/tool-parser/src/glm_xml/glm47_moe.rs +++ b/rust/src/tool-parser/src/glm_xml/glm47_moe.rs @@ -1,5 +1,5 @@ use super::{GlmXmlToolParser, Separator}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for GLM-4.7 MoE XML-style tool calls. /// @@ -22,6 +22,10 @@ impl ToolParser for Glm47MoeToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Glm47) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/hy_v3.rs b/rust/src/tool-parser/src/hy_v3.rs index 7c850d51aa12..c0cf94463486 100644 --- a/rust/src/tool-parser/src/hy_v3.rs +++ b/rust/src/tool-parser/src/hy_v3.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::{StructuralTagModel, Tool}; const TOOL_CALLS_START: &str = ""; const TOOL_CALLS_END: &str = ""; @@ -113,6 +113,10 @@ impl ToolParser for HyV3ToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::HyV3) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); diff --git a/rust/src/tool-parser/src/json/hermes.rs b/rust/src/tool-parser/src/json/hermes.rs index f6b130ec472f..04635185176c 100644 --- a/rust/src/tool-parser/src/json/hermes.rs +++ b/rust/src/tool-parser/src/json/hermes.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Hermes", @@ -45,6 +45,10 @@ impl ToolParser for HermesToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Hermes) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.inner.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/json/llama.rs b/rust/src/tool-parser/src/json/llama.rs index 36bc8a8347d9..d9456487d1f9 100644 --- a/rust/src/tool-parser/src/json/llama.rs +++ b/rust/src/tool-parser/src/json/llama.rs @@ -9,7 +9,7 @@ use super::{ argument_delta_event, tool_call_header_event, }; use crate::utils::{JsonObjectScanState, parse_buffered_event}; -use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; #[derive(Debug, Clone, PartialEq, Eq)] enum LlamaJsonMode { @@ -133,6 +133,10 @@ impl ToolParser for Llama3JsonToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Llama) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); diff --git a/rust/src/tool-parser/src/json/qwen.rs b/rust/src/tool-parser/src/json/qwen.rs index b8caff0fefd8..dd943dfffc76 100644 --- a/rust/src/tool-parser/src/json/qwen.rs +++ b/rust/src/tool-parser/src/json/qwen.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Qwen XML", @@ -47,6 +47,10 @@ impl ToolParser for Qwen3XmlToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Qwen3) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.inner.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/kimi_k2.rs b/rust/src/tool-parser/src/kimi_k2.rs index b639fdd5c331..f83611ac79db 100644 --- a/rust/src/tool-parser/src/kimi_k2.rs +++ b/rust/src/tool-parser/src/kimi_k2.rs @@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while}; use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::{StructuralTagModel, Tool}; const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>"; const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>"; @@ -147,6 +147,10 @@ impl ToolParser for KimiK2ToolParser { true } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Kimi) + } + fn tool_call_id(&self, tool_index: usize) -> Option<&str> { self.call_ids.get(&tool_index).map(String::as_str) } diff --git a/rust/src/tool-parser/src/lib.rs b/rust/src/tool-parser/src/lib.rs index b5f0b80d0452..6f785ee1d183 100644 --- a/rust/src/tool-parser/src/lib.rs +++ b/rust/src/tool-parser/src/lib.rs @@ -35,6 +35,7 @@ pub use minimax_m3::MinimaxM3ToolParser; pub use qwen_coder::Qwen3CoderToolParser; use serde::{Deserialize, Serialize}; use serde_json::Value; +pub use xgrammar_structural_tag::Model as StructuralTagModel; /// One function-style tool made available to the model. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -123,6 +124,11 @@ pub trait ToolParser: Send { false } + /// Return the xgrammar structural-tag model used for strict tool calling. + fn structural_tag_model(&self) -> Option { + None + } + /// Return the parser-provided ID for a tool call by index, if the model /// emitted one. fn tool_call_id(&self, _tool_index: usize) -> Option<&str> { diff --git a/rust/src/tool-parser/src/minimax_m2.rs b/rust/src/tool-parser/src/minimax_m2.rs index 1d7bc78987db..16e2b85525f4 100644 --- a/rust/src/tool-parser/src/minimax_m2.rs +++ b/rust/src/tool-parser/src/minimax_m2.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::{StructuralTagModel, Tool}; const TOOL_CALL_START: &str = ""; const TOOL_CALL_END: &str = ""; @@ -112,6 +112,10 @@ impl ToolParser for MinimaxM2ToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Minimax) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); diff --git a/rust/src/tool-parser/src/qwen_coder.rs b/rust/src/tool-parser/src/qwen_coder.rs index 270955aff076..5e78d7ae5200 100644 --- a/rust/src/tool-parser/src/qwen_coder.rs +++ b/rust/src/tool-parser/src/qwen_coder.rs @@ -6,7 +6,7 @@ use winnow::token::{literal, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; -use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; +use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput}; use crate::Tool; const TOOL_CALL_START: &str = ""; @@ -113,6 +113,10 @@ impl ToolParser for Qwen3CoderToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Qwen3Coder) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); @@ -236,7 +240,7 @@ mod tests { use serde_json::{Value, json}; use thiserror_ext::AsReport; - use super::{Qwen3CoderToolParser, ToolParser}; + use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser}; use crate::test_utils::{collect_stream, split_by_chars, test_tools}; use crate::{ToolParserOutput, ToolParserTestExt as _}; @@ -249,6 +253,16 @@ mod tests { format!("\n\n{params}\n\n") } + #[test] + fn qwen_coder_exposes_structural_tag_model() { + let parser = Qwen3CoderToolParser::new(&test_tools()); + + assert_eq!( + parser.structural_tag_model(), + Some(StructuralTagModel::Qwen3Coder) + ); + } + #[test] fn qwen_coder_parse_complete_without_tool_call_keeps_text() { let mut parser = Qwen3CoderToolParser::new(&test_tools());