From f5cb54b900ae388a1573d1057ac47e7ab09642b8 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 18 Jun 2026 12:30:25 +0000 Subject: [PATCH 1/6] integrate xgrammar-structural-tag Signed-off-by: Bugen Zhao --- rust/Cargo.lock | 13 ++ rust/Cargo.toml | 1 + rust/src/chat/Cargo.toml | 1 + rust/src/chat/src/error.rs | 2 + rust/src/chat/src/output/default/mod.rs | 205 +++++++++++++++++- rust/src/chat/src/request.rs | 10 +- .../routes/openai/chat_completions/convert.rs | 75 +++++++ .../openai/chat_completions/validate.rs | 129 +++++++++-- .../routes/openai/utils/structured_outputs.rs | 35 +++ rust/src/tool-parser/Cargo.toml | 1 + .../src/deepseek_dsml/deepseek_v4.rs | 18 +- rust/src/tool-parser/src/lib.rs | 6 + rust/src/tool-parser/src/qwen_coder.rs | 18 +- 13 files changed, 490 insertions(+), 24 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 60aa6c12410f..33f701d4a1a7 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -5726,6 +5726,7 @@ dependencies = [ "vllm-text", "vllm-tokenizer", "vllm-tool-parser", + "xgrammar-structural-tag", "zeromq", ] @@ -5980,6 +5981,7 @@ dependencies = [ "thiserror-ext", "tool-parser", "winnow", + "xgrammar-structural-tag", ] [[package]] @@ -6581,6 +6583,17 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" +[[package]] +name = "xgrammar-structural-tag" +version = "0.1.0+xgrammar.0.2.2.4d145cc" +source = "git+https://github.com/Inferact/xgrammar-structural-tag.git?rev=afffd968ecf3ec9ec0963953741dfdf6a00b4e10#afffd968ecf3ec9ec0963953741dfdf6a00b4e10" +dependencies = [ + "serde", + "serde_json", + "strum", + "thiserror 2.0.18", +] + [[package]] name = "y4m" version = "0.8.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 455e660bcfee..7346627abce5 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -123,6 +123,7 @@ vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } vllm-tool-parser = { path = "src/tool-parser" } winnow = "1.0.2" +xgrammar-structural-tag = { git = "https://github.com/Inferact/xgrammar-structural-tag.git", rev = "afffd968ecf3ec9ec0963953741dfdf6a00b4e10" } zeromq = { version = "0.6.0", default-features = false, features = [ "tokio-runtime", "all-transport", diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index 0523b9defe93..85368ad98b92 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -34,6 +34,7 @@ vllm-reasoning-parser.workspace = true vllm-text.workspace = true vllm-tokenizer.workspace = true vllm-tool-parser.workspace = true +xgrammar-structural-tag.workspace = true [dev-dependencies] anyhow.workspace = true diff --git a/rust/src/chat/src/error.rs b/rust/src/chat/src/error.rs index bbd99572004d..c472a65601d5 100644 --- a/rust/src/chat/src/error.rs +++ b/rust/src/chat/src/error.rs @@ -64,6 +64,8 @@ pub enum Error { StreamClosedBeforeTerminalOutput { request_id: String }, #[error("tool call stream state is inconsistent: {message}")] ToolCallStreamInvariant { message: String }, + #[error("failed to build structural tag: {message}")] + StructuralTag { message: String }, #[error(transparent)] Text(#[from] vllm_text::Error), #[error(transparent)] diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index bebcf8839d52..7a6cce271a13 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -6,9 +6,15 @@ mod tool; use std::sync::Once; use futures::{Stream, StreamExt as _}; +use thiserror_ext::AsReport; use tracing::info; use trait_set::trait_set; +use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams}; use vllm_text::tokenizer::DynTokenizer; +use xgrammar_structural_tag::{ + FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam, + build_optional_structural_tag, +}; use self::reasoning::reasoning_event_stream; use self::tool::tool_event_stream; @@ -54,8 +60,7 @@ impl DefaultChatOutputProcessor { tool_call_parser: &ParserSelection, reasoning_parser: &ParserSelection, ) -> ChatResult { - let tool_parsing_enabled = - matches!(request.tool_choice, ChatToolChoice::Auto) && !request.tools.is_empty(); + let tool_parsing_enabled = request.tool_parsing_enabled(); let tool_parser = if tool_parsing_enabled { Some(Self::resolve_tool_parser( request, @@ -115,6 +120,8 @@ impl DefaultChatOutputProcessor { request.decode_options.skip_special_tokens = false; } + apply_structural_tag_constraint(request, parser.as_ref())?; + TOOL_PARSER_LOG_ONCE.call_once(|| info!(parser_name, "using tool parser")); Ok(parser) } @@ -151,6 +158,62 @@ impl DefaultChatOutputProcessor { static TOOL_PARSER_LOG_ONCE: Once = Once::new(); static REASONING_PARSER_LOG_ONCE: Once = Once::new(); +fn apply_structural_tag_constraint( + request: &mut ChatRequest, + parser: &dyn ToolParser, +) -> ChatResult<()> { + let Some(model) = parser.structural_tag_model() else { + return Ok(()); + }; + let Some(tool_choice) = structural_tag_tool_choice(request) else { + return Ok(()); + }; + + let tools = request + .tools + .iter() + .map(|tool| { + ToolParam::Function(FunctionToolParam::new(FunctionDefinition { + name: tool.name.clone(), + description: tool.description.clone(), + parameters: Some(tool.parameters.clone()), + strict: tool.strict, + })) + }) + .collect::>(); + + let tag = + build_optional_structural_tag(model, &tools, tool_choice, false).map_err(|error| { + Error::StructuralTag { + message: error.to_report_string(), + } + })?; + let Some(tag) = tag else { + return Ok(()); + }; + let structural_tag = tag.to_json_string().map_err(|error| Error::StructuralTag { + message: error.to_report_string(), + })?; + + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + structural_tag: Some(structural_tag), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + Ok(()) +} + +fn structural_tag_tool_choice(request: &ChatRequest) -> Option { + match &request.tool_choice { + ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => { + Some(StructuralTagToolChoice::auto()) + } + ChatToolChoice::Auto | ChatToolChoice::None => None, + ChatToolChoice::Required => Some(StructuralTagToolChoice::required()), + ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())), + } +} + impl ChatOutputProcessor for DefaultChatOutputProcessor { /// Transforms a raw generate-output token stream into structured chat /// events through three sequential stages once text decoding has @@ -167,3 +230,141 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor { Ok(structured.boxed()) } } + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + use vllm_tool_parser::{ + Result as ToolParserResult, StructuralTagModel, Tool, ToolParserOutput, + }; + + use super::*; + + struct StructuralTagParser; + + impl ToolParser for StructuralTagParser { + fn create(_tools: &[Tool]) -> ToolParserResult> + where + Self: Sized + 'static, + { + Ok(Box::new(Self)) + } + + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Qwen3Coder) + } + + fn parse_into( + &mut self, + _chunk: &str, + _output: &mut ToolParserOutput, + ) -> ToolParserResult<()> { + Ok(()) + } + + fn finish(&mut self) -> ToolParserResult { + Ok(ToolParserOutput::default()) + } + + fn reset(&mut self) -> String { + String::new() + } + } + + fn chat_tool(name: &str, strict: Option) -> Tool { + Tool { + name: name.to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": { + "query": { "type": "string" } + }, + "required": ["query"] + }), + strict, + } + } + + fn request(tool_choice: ChatToolChoice, tools: Vec) -> ChatRequest { + ChatRequest { + tool_choice, + tools, + ..ChatRequest::for_test() + } + } + + fn structural_tag_value(request: &ChatRequest) -> Value { + let params = request + .sampling_params + .structured_outputs + .as_ref() + .expect("structured outputs should be set"); + assert_eq!(params.backend, StructuredOutputBackend::Xgrammar); + serde_json::from_str( + params.structural_tag.as_deref().expect("structural_tag should be set"), + ) + .expect("structural_tag should be valid JSON") + } + + #[test] + fn auto_strict_tool_choice_builds_structural_tag() { + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn auto_non_strict_tool_choice_skips_structural_tag() { + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag decision should succeed"); + + assert!(request.sampling_params.structured_outputs.is_none()); + } + + #[test] + fn required_tool_choice_builds_structural_tag_without_strict_tools() { + let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn named_tool_choice_builds_structural_tag_for_named_tool_only() { + let mut request = request( + ChatToolChoice::Function { + name: "lookup".to_string(), + }, + vec![chat_tool("search", None), chat_tool("lookup", None)], + ); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request).to_string(); + assert!(tag.contains("lookup")); + assert!(!tag.contains("search")); + } + + #[test] + fn none_tool_choice_skips_structural_tag() { + let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag decision should succeed"); + + assert!(request.sampling_params.structured_outputs.is_none()); + } +} diff --git a/rust/src/chat/src/request.rs b/rust/src/chat/src/request.rs index 7b9ae5f663e0..51b2efebd417 100644 --- a/rust/src/chat/src/request.rs +++ b/rust/src/chat/src/request.rs @@ -382,12 +382,16 @@ impl ChatOptions { } /// Tool-choice semantics supported by `vllm-chat`. -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] pub enum ChatToolChoice { - Auto, #[default] None, + Auto, + Required, + Function { + name: String, + }, } /// One chat request ready to be rendered into a prompt and lowered into a @@ -486,7 +490,7 @@ impl ChatRequest { /// Return true if this request should enable tool parsing based on the tool /// choice and tool list. pub(crate) fn tool_parsing_enabled(&self) -> bool { - matches!(self.tool_choice, ChatToolChoice::Auto) && !self.tools.is_empty() + !matches!(self.tool_choice, ChatToolChoice::None) && !self.tools.is_empty() } /// Return the request-level thinking toggle when explicitly requested. diff --git a/rust/src/server/src/routes/openai/chat_completions/convert.rs b/rust/src/server/src/routes/openai/chat_completions/convert.rs index 0462294b0c29..2d64d194bf2e 100644 --- a/rust/src/server/src/routes/openai/chat_completions/convert.rs +++ b/rust/src/server/src/routes/openai/chat_completions/convert.rs @@ -361,6 +361,13 @@ fn convert_tool_choice(tool_choice: Option<&ToolChoice>) -> Result Ok(ChatToolChoice::Auto), Some(ToolChoice::Value(ToolChoiceValue::None)) => Ok(ChatToolChoice::None), + Some(ToolChoice::Value(ToolChoiceValue::Required)) => Ok(ChatToolChoice::Required), + Some(ToolChoice::Function { + tool_type, + function, + }) if tool_type == "function" => Ok(ChatToolChoice::Function { + name: function.name.clone(), + }), _ => bail_invalid_request!("tool_choice={:?} is not supported yet.", tool_choice), } } @@ -962,6 +969,74 @@ mod tests { assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::None); } + #[test] + fn prepare_chat_request_lowers_required_tool_choice() { + let request = ChatCompletionRequest { + tools: Some(vec![Tool { + tool_type: "function".to_string(), + function: Function { + name: "get_weather".to_string(), + description: Some("Get weather".to_string()), + parameters: json!({ + "type": "object", + "properties": {"city": {"type": "string"}}, + }), + strict: None, + }, + }]), + tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)), + ..base_request() + }; + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert_eq!(prepared.chat_request.tool_choice, ChatToolChoice::Required); + } + + #[test] + fn prepare_chat_request_lowers_named_function_tool_choice() { + let request = ChatCompletionRequest { + tools: Some(vec![Tool { + tool_type: "function".to_string(), + function: Function { + name: "get_weather".to_string(), + description: Some("Get weather".to_string()), + parameters: json!({ + "type": "object", + "properties": {"city": {"type": "string"}}, + }), + strict: None, + }, + }]), + tool_choice: Some(ToolChoice::Function { + tool_type: "function".to_string(), + function: crate::routes::openai::utils::types::FunctionChoice { + name: "get_weather".to_string(), + }, + }), + ..base_request() + }; + + let prepared = prepare_chat_request( + request, + &served(&["Qwen/Qwen1.5-0.5B-Chat"]), + ResolvedRequestContext::default(), + ) + .expect("request is valid"); + + assert_eq!( + prepared.chat_request.tool_choice, + ChatToolChoice::Function { + name: "get_weather".to_string(), + } + ); + } + #[test] fn prepare_chat_request_lowers_logprobs_fields() { let request = ChatCompletionRequest { diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index 2e789573d219..90b82206ca36 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -1,5 +1,6 @@ use super::types::ChatCompletionRequest; use crate::error::{ApiError, bail_invalid_request}; +use crate::routes::openai::utils::structured_outputs::ResponseFormat; use crate::routes::openai::utils::types::{ChatMessage, Tool, ToolChoice, ToolChoiceValue}; /// Enforce the minimal compatibility contract for the Rust OpenAI server. @@ -60,19 +61,10 @@ pub(super) fn validate_request_compat( if let Some(tool_choice) = &request.tool_choice { match tool_choice { - ToolChoice::Value(ToolChoiceValue::Auto | ToolChoiceValue::None) => {} - ToolChoice::Value(ToolChoiceValue::Required) => { - bail_invalid_request!( - param = "tool_choice", - "tool_choice=required is not supported yet." - ); - } - ToolChoice::Function { .. } => { - bail_invalid_request!( - param = "tool_choice", - "Named function tool_choice is not supported yet." - ); - } + ToolChoice::Value( + ToolChoiceValue::Auto | ToolChoiceValue::None | ToolChoiceValue::Required, + ) + | ToolChoice::Function { .. } => {} ToolChoice::AllowedTools { .. } => { bail_invalid_request!( param = "tool_choice", @@ -82,6 +74,20 @@ pub(super) fn validate_request_compat( } } + if has_explicit_structured_output_guidance(request) + && tool_choice_uses_structural_tag_constraint(request) + { + let param = request + .structured_outputs + .as_ref() + .map(|_| "structured_outputs") + .unwrap_or("response_format"); + bail_invalid_request!( + param = param, + "Explicit structured output guidance cannot be combined with strict tool calling." + ); + } + if request.use_beam_search { bail_invalid_request!( param = "use_beam_search", @@ -127,6 +133,30 @@ pub(super) fn validate_request_compat( Ok(()) } +fn has_explicit_structured_output_guidance(request: &ChatCompletionRequest) -> bool { + request.structured_outputs.is_some() + || matches!( + request.response_format.as_ref(), + Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. }) + ) +} + +fn tool_choice_uses_structural_tag_constraint(request: &ChatCompletionRequest) -> bool { + match &request.tool_choice { + Some(ToolChoice::Value(ToolChoiceValue::Required)) | Some(ToolChoice::Function { .. }) => { + true + } + Some(ToolChoice::Value(ToolChoiceValue::Auto)) => request + .tools + .as_ref() + .is_some_and(|tools| tools.iter().any(|tool| tool.function.strict == Some(true))), + Some(ToolChoice::Value(ToolChoiceValue::None)) | Some(ToolChoice::AllowedTools { .. }) => { + false + } + None => false, + } +} + /// Reject one option unless it is entirely absent. fn reject_non_default( value: Option<&T>, @@ -359,24 +389,93 @@ mod tests { } #[test] - fn validate_request_compat_rejects_required_and_named_tool_choices() { + fn validate_request_compat_accepts_required_and_named_tool_choices() { + let tools = Some(vec![Tool { + tool_type: "function".to_string(), + function: Function { + name: "tool".to_string(), + description: None, + parameters: json!({}), + strict: None, + }, + }]); let required = ChatCompletionRequest { tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)), + tools: tools.clone(), + ..base_request() + }; + validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) + .expect("tool_choice=required should be accepted"); + + let named = ChatCompletionRequest { + tool_choice: Some(ToolChoice::Function { + tool_type: "function".to_string(), + function: FunctionChoice { + name: "tool".to_string(), + }, + }), + tools, ..base_request() }; - assert!(validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); + validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) + .expect("named function tool_choice should be accepted"); + } + #[test] + fn validate_request_compat_rejects_guidance_with_strict_tool_calling() { + let strict_tools = Some(vec![Tool { + tool_type: "function".to_string(), + function: Function { + name: "tool".to_string(), + description: None, + parameters: json!({}), + strict: Some(true), + }, + }]); let named = ChatCompletionRequest { + response_format: Some(ResponseFormat::JsonObject), tool_choice: Some(ToolChoice::Function { tool_type: "function".to_string(), function: FunctionChoice { name: "tool".to_string(), }, }), + tools: strict_tools.clone(), ..base_request() }; assert!(validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); + let auto = ChatCompletionRequest { + structured_outputs: Some(json!({"json": {"type": "object"}})), + tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Auto)), + tools: strict_tools, + ..base_request() + }; + assert!(validate_request_compat(&auto, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); + } + + #[test] + fn validate_request_compat_allows_text_response_format_with_strict_tool_calling() { + let request = ChatCompletionRequest { + response_format: Some(ResponseFormat::Text), + tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)), + tools: Some(vec![Tool { + tool_type: "function".to_string(), + function: Function { + name: "tool".to_string(), + description: None, + parameters: json!({}), + strict: Some(true), + }, + }]), + ..base_request() + }; + validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) + .expect("response_format=text should not conflict with strict tool calling"); + } + + #[test] + fn validate_request_compat_rejects_allowed_tools_choice() { let allowed_tools = ChatCompletionRequest { tool_choice: Some(ToolChoice::AllowedTools { tool_type: "allowed_tools".to_string(), diff --git a/rust/src/server/src/routes/openai/utils/structured_outputs.rs b/rust/src/server/src/routes/openai/utils/structured_outputs.rs index e974c836bb5e..12b265cf0d55 100644 --- a/rust/src/server/src/routes/openai/utils/structured_outputs.rs +++ b/rust/src/server/src/routes/openai/utils/structured_outputs.rs @@ -53,6 +53,12 @@ pub enum ResponseFormat { fn deserialize_structured_outputs( raw: &serde_json::Value, ) -> Result { + if raw.get("_backend").is_some() { + return Err(ApiError::invalid_request( + "request-level structured output backend selection is not supported", + Some("structured_outputs._backend"), + )); + } serde_json::from_value(raw.clone()).map_err(|e| { ApiError::invalid_request( format!("invalid structured_outputs: {e}"), @@ -130,3 +136,32 @@ pub fn convert_from_response_format_value( })?; convert_from_response_format(Some(&fmt), &None) } + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::convert_from_response_format; + + #[test] + fn rejects_request_level_structured_output_backend() { + let err = convert_from_response_format( + None, + &Some(json!({ + "json_object": true, + "_backend": "guidance", + })), + ) + .expect_err("request-level backend selection should be rejected"); + + let response = err.to_error_response(); + assert_eq!( + response.error.message, + "request-level structured output backend selection is not supported" + ); + assert_eq!( + response.error.param.as_deref(), + Some("structured_outputs._backend") + ); + } +} diff --git a/rust/src/tool-parser/Cargo.toml b/rust/src/tool-parser/Cargo.toml index 0bc7010b75ba..c4363906aa05 100644 --- a/rust/src/tool-parser/Cargo.toml +++ b/rust/src/tool-parser/Cargo.toml @@ -14,6 +14,7 @@ serde_json.workspace = true thiserror.workspace = true thiserror-ext.workspace = true winnow.workspace = true +xgrammar-structural-tag.workspace = true [dev-dependencies] criterion.workspace = true diff --git a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs b/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs index ba01b1b586a3..9047b24ced70 100644 --- a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs +++ b/rust/src/tool-parser/src/deepseek_dsml/deepseek_v4.rs @@ -1,5 +1,5 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V4 models. /// @@ -47,6 +47,10 @@ impl ToolParser for DeepSeekV4ToolParser { true } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::DeepSeekV4) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } @@ -65,8 +69,8 @@ mod tests { use serde_json::{Value, json}; use super::DeepSeekV4ToolParser; - use crate::ToolParserTestExt as _; use crate::test_utils::{collect_stream, test_tools}; + use crate::{StructuralTagModel, ToolParser, ToolParserTestExt as _}; fn build_tool_call(function_name: &str, params: &[(&str, &str)]) -> String { let params = params @@ -83,6 +87,16 @@ mod tests { ) } + #[test] + fn deepseek_v4_exposes_structural_tag_model() { + let parser = DeepSeekV4ToolParser::new(&test_tools()); + + assert_eq!( + parser.structural_tag_model(), + Some(StructuralTagModel::DeepSeekV4) + ); + } + #[test] fn deepseek_v4_parse_complete_reuses_dsml_parser_with_tool_calls_token() { let mut parser = DeepSeekV4ToolParser::new(&test_tools()); diff --git a/rust/src/tool-parser/src/lib.rs b/rust/src/tool-parser/src/lib.rs index b5f0b80d0452..6f785ee1d183 100644 --- a/rust/src/tool-parser/src/lib.rs +++ b/rust/src/tool-parser/src/lib.rs @@ -35,6 +35,7 @@ pub use minimax_m3::MinimaxM3ToolParser; pub use qwen_coder::Qwen3CoderToolParser; use serde::{Deserialize, Serialize}; use serde_json::Value; +pub use xgrammar_structural_tag::Model as StructuralTagModel; /// One function-style tool made available to the model. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -123,6 +124,11 @@ pub trait ToolParser: Send { false } + /// Return the xgrammar structural-tag model used for strict tool calling. + fn structural_tag_model(&self) -> Option { + None + } + /// Return the parser-provided ID for a tool call by index, if the model /// emitted one. fn tool_call_id(&self, _tool_index: usize) -> Option<&str> { diff --git a/rust/src/tool-parser/src/qwen_coder.rs b/rust/src/tool-parser/src/qwen_coder.rs index 270955aff076..5e78d7ae5200 100644 --- a/rust/src/tool-parser/src/qwen_coder.rs +++ b/rust/src/tool-parser/src/qwen_coder.rs @@ -6,7 +6,7 @@ use winnow::token::{literal, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; -use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; +use super::{Result, StructuralTagModel, ToolCallDelta, ToolParser, ToolParserOutput}; use crate::Tool; const TOOL_CALL_START: &str = ""; @@ -113,6 +113,10 @@ impl ToolParser for Qwen3CoderToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Qwen3Coder) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); @@ -236,7 +240,7 @@ mod tests { use serde_json::{Value, json}; use thiserror_ext::AsReport; - use super::{Qwen3CoderToolParser, ToolParser}; + use super::{Qwen3CoderToolParser, StructuralTagModel, ToolParser}; use crate::test_utils::{collect_stream, split_by_chars, test_tools}; use crate::{ToolParserOutput, ToolParserTestExt as _}; @@ -249,6 +253,16 @@ mod tests { format!("\n\n{params}\n\n") } + #[test] + fn qwen_coder_exposes_structural_tag_model() { + let parser = Qwen3CoderToolParser::new(&test_tools()); + + assert_eq!( + parser.structural_tag_model(), + Some(StructuralTagModel::Qwen3Coder) + ); + } + #[test] fn qwen_coder_parse_complete_without_tool_call_keeps_text() { let mut parser = Qwen3CoderToolParser::new(&test_tools()); From 43f3b0d7a9b605481e1f4c3e06b846c0ed1a52cb Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Thu, 18 Jun 2026 14:43:20 +0000 Subject: [PATCH 2/6] associate more models Signed-off-by: Bugen Zhao --- rust/src/chat/src/output/default/mod.rs | 204 +---------------- .../chat/src/output/default/structural_tag.rs | 207 ++++++++++++++++++ .../src/deepseek_dsml/deepseek_v32.rs | 6 +- .../src/deepseek_json/deepseek_v3.rs | 6 +- .../src/deepseek_json/deepseek_v31.rs | 6 +- rust/src/tool-parser/src/glm_xml/glm47_moe.rs | 6 +- rust/src/tool-parser/src/hy_v3.rs | 6 +- rust/src/tool-parser/src/json/hermes.rs | 6 +- rust/src/tool-parser/src/json/llama.rs | 6 +- rust/src/tool-parser/src/json/qwen.rs | 6 +- rust/src/tool-parser/src/kimi_k2.rs | 6 +- rust/src/tool-parser/src/minimax_m2.rs | 6 +- 12 files changed, 260 insertions(+), 211 deletions(-) create mode 100644 rust/src/chat/src/output/default/structural_tag.rs diff --git a/rust/src/chat/src/output/default/mod.rs b/rust/src/chat/src/output/default/mod.rs index 7a6cce271a13..dbc9cc05a517 100644 --- a/rust/src/chat/src/output/default/mod.rs +++ b/rust/src/chat/src/output/default/mod.rs @@ -1,22 +1,18 @@ //! Default output processing pipeline. mod reasoning; +mod structural_tag; mod tool; use std::sync::Once; use futures::{Stream, StreamExt as _}; -use thiserror_ext::AsReport; use tracing::info; use trait_set::trait_set; -use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams}; use vllm_text::tokenizer::DynTokenizer; -use xgrammar_structural_tag::{ - FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam, - build_optional_structural_tag, -}; use self::reasoning::reasoning_event_stream; +use self::structural_tag::apply_structural_tag_constraint; use self::tool::tool_event_stream; use super::structured::structured_chat_event_stream; use crate::error::Result; @@ -27,7 +23,7 @@ use crate::output::{ use crate::parser::ParserSelection; use crate::parser::reasoning::{ReasoningParser, ReasoningParserFactory}; use crate::parser::tool::{ToolParser, ToolParserFactory}; -use crate::request::{ChatRequest, ChatToolChoice}; +use crate::request::ChatRequest; use crate::{Error, Result as ChatResult}; trait_set! { @@ -158,62 +154,6 @@ impl DefaultChatOutputProcessor { static TOOL_PARSER_LOG_ONCE: Once = Once::new(); static REASONING_PARSER_LOG_ONCE: Once = Once::new(); -fn apply_structural_tag_constraint( - request: &mut ChatRequest, - parser: &dyn ToolParser, -) -> ChatResult<()> { - let Some(model) = parser.structural_tag_model() else { - return Ok(()); - }; - let Some(tool_choice) = structural_tag_tool_choice(request) else { - return Ok(()); - }; - - let tools = request - .tools - .iter() - .map(|tool| { - ToolParam::Function(FunctionToolParam::new(FunctionDefinition { - name: tool.name.clone(), - description: tool.description.clone(), - parameters: Some(tool.parameters.clone()), - strict: tool.strict, - })) - }) - .collect::>(); - - let tag = - build_optional_structural_tag(model, &tools, tool_choice, false).map_err(|error| { - Error::StructuralTag { - message: error.to_report_string(), - } - })?; - let Some(tag) = tag else { - return Ok(()); - }; - let structural_tag = tag.to_json_string().map_err(|error| Error::StructuralTag { - message: error.to_report_string(), - })?; - - request.sampling_params.structured_outputs = Some(StructuredOutputsParams { - structural_tag: Some(structural_tag), - backend: StructuredOutputBackend::Xgrammar, - ..Default::default() - }); - Ok(()) -} - -fn structural_tag_tool_choice(request: &ChatRequest) -> Option { - match &request.tool_choice { - ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => { - Some(StructuralTagToolChoice::auto()) - } - ChatToolChoice::Auto | ChatToolChoice::None => None, - ChatToolChoice::Required => Some(StructuralTagToolChoice::required()), - ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())), - } -} - impl ChatOutputProcessor for DefaultChatOutputProcessor { /// Transforms a raw generate-output token stream into structured chat /// events through three sequential stages once text decoding has @@ -230,141 +170,3 @@ impl ChatOutputProcessor for DefaultChatOutputProcessor { Ok(structured.boxed()) } } - -#[cfg(test)] -mod tests { - use serde_json::{Value, json}; - use vllm_tool_parser::{ - Result as ToolParserResult, StructuralTagModel, Tool, ToolParserOutput, - }; - - use super::*; - - struct StructuralTagParser; - - impl ToolParser for StructuralTagParser { - fn create(_tools: &[Tool]) -> ToolParserResult> - where - Self: Sized + 'static, - { - Ok(Box::new(Self)) - } - - fn structural_tag_model(&self) -> Option { - Some(StructuralTagModel::Qwen3Coder) - } - - fn parse_into( - &mut self, - _chunk: &str, - _output: &mut ToolParserOutput, - ) -> ToolParserResult<()> { - Ok(()) - } - - fn finish(&mut self) -> ToolParserResult { - Ok(ToolParserOutput::default()) - } - - fn reset(&mut self) -> String { - String::new() - } - } - - fn chat_tool(name: &str, strict: Option) -> Tool { - Tool { - name: name.to_string(), - description: None, - parameters: json!({ - "type": "object", - "properties": { - "query": { "type": "string" } - }, - "required": ["query"] - }), - strict, - } - } - - fn request(tool_choice: ChatToolChoice, tools: Vec) -> ChatRequest { - ChatRequest { - tool_choice, - tools, - ..ChatRequest::for_test() - } - } - - fn structural_tag_value(request: &ChatRequest) -> Value { - let params = request - .sampling_params - .structured_outputs - .as_ref() - .expect("structured outputs should be set"); - assert_eq!(params.backend, StructuredOutputBackend::Xgrammar); - serde_json::from_str( - params.structural_tag.as_deref().expect("structural_tag should be set"), - ) - .expect("structural_tag should be valid JSON") - } - - #[test] - fn auto_strict_tool_choice_builds_structural_tag() { - let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); - - apply_structural_tag_constraint(&mut request, &StructuralTagParser) - .expect("structural tag should build"); - - let tag = structural_tag_value(&request); - assert_eq!(tag["type"], "structural_tag"); - assert!(tag.to_string().contains("search")); - } - - #[test] - fn auto_non_strict_tool_choice_skips_structural_tag() { - let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]); - - apply_structural_tag_constraint(&mut request, &StructuralTagParser) - .expect("structural tag decision should succeed"); - - assert!(request.sampling_params.structured_outputs.is_none()); - } - - #[test] - fn required_tool_choice_builds_structural_tag_without_strict_tools() { - let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); - - apply_structural_tag_constraint(&mut request, &StructuralTagParser) - .expect("structural tag should build"); - - let tag = structural_tag_value(&request); - assert_eq!(tag["type"], "structural_tag"); - assert!(tag.to_string().contains("search")); - } - - #[test] - fn named_tool_choice_builds_structural_tag_for_named_tool_only() { - let mut request = request( - ChatToolChoice::Function { - name: "lookup".to_string(), - }, - vec![chat_tool("search", None), chat_tool("lookup", None)], - ); - - apply_structural_tag_constraint(&mut request, &StructuralTagParser) - .expect("structural tag should build"); - - let tag = structural_tag_value(&request).to_string(); - assert!(tag.contains("lookup")); - assert!(!tag.contains("search")); - } - - #[test] - fn none_tool_choice_skips_structural_tag() { - let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); - - apply_structural_tag_constraint(&mut request, &StructuralTagParser) - .expect("structural tag decision should succeed"); - - assert!(request.sampling_params.structured_outputs.is_none()); - } -} diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs new file mode 100644 index 000000000000..95d300310591 --- /dev/null +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -0,0 +1,207 @@ +//! Applies xgrammar structural-tag constraints for strict tool calling. + +use thiserror_ext::AsReport; +use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams}; +use xgrammar_structural_tag::{ + FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam, + build_optional_structural_tag, +}; + +use crate::parser::tool::ToolParser; +use crate::request::{ChatRequest, ChatToolChoice}; +use crate::{Error, Result as ChatResult}; + +pub(super) fn apply_structural_tag_constraint( + request: &mut ChatRequest, + parser: &dyn ToolParser, +) -> ChatResult<()> { + let Some(model) = parser.structural_tag_model() else { + return Ok(()); + }; + let Some(tool_choice) = structural_tag_tool_choice(request) else { + return Ok(()); + }; + + let tools = request + .tools + .iter() + .map(|tool| { + ToolParam::Function(FunctionToolParam::new(FunctionDefinition { + name: tool.name.clone(), + description: tool.description.clone(), + parameters: Some(tool.parameters.clone()), + strict: tool.strict, + })) + }) + .collect::>(); + + let tag = + build_optional_structural_tag(model, &tools, tool_choice, false).map_err(|error| { + Error::StructuralTag { + message: error.to_report_string(), + } + })?; + let Some(tag) = tag else { + return Ok(()); + }; + let structural_tag = tag.to_json_string().map_err(|error| Error::StructuralTag { + message: error.to_report_string(), + })?; + + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + structural_tag: Some(structural_tag), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + Ok(()) +} + +fn structural_tag_tool_choice(request: &ChatRequest) -> Option { + match &request.tool_choice { + ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => { + Some(StructuralTagToolChoice::auto()) + } + ChatToolChoice::Auto | ChatToolChoice::None => None, + ChatToolChoice::Required => Some(StructuralTagToolChoice::required()), + ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())), + } +} + +#[cfg(test)] +mod tests { + use serde_json::{Value, json}; + use vllm_engine_core_client::protocol::StructuredOutputBackend; + use vllm_tool_parser::{ + Result as ToolParserResult, StructuralTagModel, Tool, ToolParserOutput, + }; + + use super::*; + + struct StructuralTagParser; + + impl ToolParser for StructuralTagParser { + fn create(_tools: &[Tool]) -> ToolParserResult> + where + Self: Sized + 'static, + { + Ok(Box::new(Self)) + } + + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Qwen3Coder) + } + + fn parse_into( + &mut self, + _chunk: &str, + _output: &mut ToolParserOutput, + ) -> ToolParserResult<()> { + Ok(()) + } + + fn finish(&mut self) -> ToolParserResult { + Ok(ToolParserOutput::default()) + } + + fn reset(&mut self) -> String { + String::new() + } + } + + fn chat_tool(name: &str, strict: Option) -> Tool { + Tool { + name: name.to_string(), + description: None, + parameters: json!({ + "type": "object", + "properties": { + "query": { "type": "string" } + }, + "required": ["query"] + }), + strict, + } + } + + fn request(tool_choice: ChatToolChoice, tools: Vec) -> ChatRequest { + ChatRequest { + tool_choice, + tools, + ..ChatRequest::for_test() + } + } + + fn structural_tag_value(request: &ChatRequest) -> Value { + let params = request + .sampling_params + .structured_outputs + .as_ref() + .expect("structured outputs should be set"); + assert_eq!(params.backend, StructuredOutputBackend::Xgrammar); + serde_json::from_str( + params.structural_tag.as_deref().expect("structural_tag should be set"), + ) + .expect("structural_tag should be valid JSON") + } + + #[test] + fn auto_strict_tool_choice_builds_structural_tag() { + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn auto_non_strict_tool_choice_skips_structural_tag() { + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag decision should succeed"); + + assert!(request.sampling_params.structured_outputs.is_none()); + } + + #[test] + fn required_tool_choice_builds_structural_tag_without_strict_tools() { + let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + + #[test] + fn named_tool_choice_builds_structural_tag_for_named_tool_only() { + let mut request = request( + ChatToolChoice::Function { + name: "lookup".to_string(), + }, + vec![chat_tool("search", None), chat_tool("lookup", None)], + ); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag should build"); + + let tag = structural_tag_value(&request).to_string(); + assert!(tag.contains("lookup")); + assert!(!tag.contains("search")); + } + + #[test] + fn none_tool_choice_skips_structural_tag() { + let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); + + apply_structural_tag_constraint(&mut request, &StructuralTagParser) + .expect("structural tag decision should succeed"); + + assert!(request.sampling_params.structured_outputs.is_none()); + } +} diff --git a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs b/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs index abca33336a7b..bc636c6035a1 100644 --- a/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs +++ b/rust/src/tool-parser/src/deepseek_dsml/deepseek_v32.rs @@ -1,5 +1,5 @@ use super::{DeepSeekDsmlToolParser, DsmlTokens}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3.2 models. /// @@ -44,6 +44,10 @@ impl ToolParser for DeepSeekV32ToolParser { true } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::DeepSeekV32) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs b/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs index 9c8a2a5c5850..6d6062432ab5 100644 --- a/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs +++ b/rust/src/tool-parser/src/deepseek_json/deepseek_v3.rs @@ -1,5 +1,5 @@ use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3 JSON-fenced tool calls. /// @@ -32,6 +32,10 @@ impl ToolParser for DeepSeekV3ToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::DeepSeekR1) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs b/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs index d16ffd7de6e7..33b362439a86 100644 --- a/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs +++ b/rust/src/tool-parser/src/deepseek_json/deepseek_v31.rs @@ -1,5 +1,5 @@ use super::{DeepSeekJsonFormat, DeepSeekJsonToolParser}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for DeepSeek V3.1 raw JSON tool calls. /// @@ -28,6 +28,10 @@ impl ToolParser for DeepSeekV31ToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::DeepSeekV31) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/glm_xml/glm47_moe.rs b/rust/src/tool-parser/src/glm_xml/glm47_moe.rs index 3d1c38d55f7a..74afd6c250b4 100644 --- a/rust/src/tool-parser/src/glm_xml/glm47_moe.rs +++ b/rust/src/tool-parser/src/glm_xml/glm47_moe.rs @@ -1,5 +1,5 @@ use super::{GlmXmlToolParser, Separator}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; /// Tool parser for GLM-4.7 MoE XML-style tool calls. /// @@ -22,6 +22,10 @@ impl ToolParser for Glm47MoeToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Glm47) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.0.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/hy_v3.rs b/rust/src/tool-parser/src/hy_v3.rs index 7c850d51aa12..c0cf94463486 100644 --- a/rust/src/tool-parser/src/hy_v3.rs +++ b/rust/src/tool-parser/src/hy_v3.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::{StructuralTagModel, Tool}; const TOOL_CALLS_START: &str = ""; const TOOL_CALLS_END: &str = ""; @@ -113,6 +113,10 @@ impl ToolParser for HyV3ToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::HyV3) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); diff --git a/rust/src/tool-parser/src/json/hermes.rs b/rust/src/tool-parser/src/json/hermes.rs index f6b130ec472f..04635185176c 100644 --- a/rust/src/tool-parser/src/json/hermes.rs +++ b/rust/src/tool-parser/src/json/hermes.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; const HERMES_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Hermes", @@ -45,6 +45,10 @@ impl ToolParser for HermesToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Hermes) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.inner.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/json/llama.rs b/rust/src/tool-parser/src/json/llama.rs index 36bc8a8347d9..d9456487d1f9 100644 --- a/rust/src/tool-parser/src/json/llama.rs +++ b/rust/src/tool-parser/src/json/llama.rs @@ -9,7 +9,7 @@ use super::{ argument_delta_event, tool_call_header_event, }; use crate::utils::{JsonObjectScanState, parse_buffered_event}; -use crate::{Result, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolCallDelta, ToolParser, ToolParserOutput}; #[derive(Debug, Clone, PartialEq, Eq)] enum LlamaJsonMode { @@ -133,6 +133,10 @@ impl ToolParser for Llama3JsonToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Llama) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); diff --git a/rust/src/tool-parser/src/json/qwen.rs b/rust/src/tool-parser/src/json/qwen.rs index b8caff0fefd8..dd943dfffc76 100644 --- a/rust/src/tool-parser/src/json/qwen.rs +++ b/rust/src/tool-parser/src/json/qwen.rs @@ -1,5 +1,5 @@ use super::{JsonToolCallConfig, JsonToolCallParser, JsonToolCallWhitespace}; -use crate::{Result, Tool, ToolParser, ToolParserOutput}; +use crate::{Result, StructuralTagModel, Tool, ToolParser, ToolParserOutput}; const QWEN_XML_CONFIG: JsonToolCallConfig = JsonToolCallConfig { parser_name: "Qwen XML", @@ -47,6 +47,10 @@ impl ToolParser for Qwen3XmlToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Qwen3) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.inner.parse_into(chunk, output) } diff --git a/rust/src/tool-parser/src/kimi_k2.rs b/rust/src/tool-parser/src/kimi_k2.rs index b639fdd5c331..f83611ac79db 100644 --- a/rust/src/tool-parser/src/kimi_k2.rs +++ b/rust/src/tool-parser/src/kimi_k2.rs @@ -8,7 +8,7 @@ use winnow::token::{literal, rest, take_until, take_while}; use super::utils::{JsonObjectScanState, parse_buffered_event, safe_text_len, take_json_object}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::{StructuralTagModel, Tool}; const TOOL_CALLS_START: &str = "<|tool_calls_section_begin|>"; const TOOL_CALLS_END: &str = "<|tool_calls_section_end|>"; @@ -147,6 +147,10 @@ impl ToolParser for KimiK2ToolParser { true } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Kimi) + } + fn tool_call_id(&self, tool_index: usize) -> Option<&str> { self.call_ids.get(&tool_index).map(String::as_str) } diff --git a/rust/src/tool-parser/src/minimax_m2.rs b/rust/src/tool-parser/src/minimax_m2.rs index 1d7bc78987db..16e2b85525f4 100644 --- a/rust/src/tool-parser/src/minimax_m2.rs +++ b/rust/src/tool-parser/src/minimax_m2.rs @@ -7,7 +7,7 @@ use winnow::token::{literal, rest, take_until}; use super::parameters::ToolSchemas; use super::utils::{MarkerScanState, parse_buffered_event, safe_text_len, take_until_marker}; use super::{Result, ToolCallDelta, ToolParser, ToolParserOutput}; -use crate::Tool; +use crate::{StructuralTagModel, Tool}; const TOOL_CALL_START: &str = ""; const TOOL_CALL_END: &str = ""; @@ -112,6 +112,10 @@ impl ToolParser for MinimaxM2ToolParser { Ok(Box::new(Self::new(tools))) } + fn structural_tag_model(&self) -> Option { + Some(StructuralTagModel::Minimax) + } + fn parse_into(&mut self, chunk: &str, output: &mut ToolParserOutput) -> Result<()> { self.buffer.push_str(chunk); From a100cae0761255ac113775066d94564c5472baa9 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Mon, 22 Jun 2026 19:41:05 +0800 Subject: [PATCH 3/6] simplify impl Signed-off-by: Bugen Zhao --- .../chat/src/output/default/structural_tag.rs | 54 ++---- .../openai/chat_completions/validate.rs | 161 +----------------- .../routes/openai/utils/structured_outputs.rs | 35 ---- 3 files changed, 17 insertions(+), 233 deletions(-) diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs index 95d300310591..51013e6c6c41 100644 --- a/rust/src/chat/src/output/default/structural_tag.rs +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -71,43 +71,10 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option ToolParserResult> - where - Self: Sized + 'static, - { - Ok(Box::new(Self)) - } - - fn structural_tag_model(&self) -> Option { - Some(StructuralTagModel::Qwen3Coder) - } - - fn parse_into( - &mut self, - _chunk: &str, - _output: &mut ToolParserOutput, - ) -> ToolParserResult<()> { - Ok(()) - } - - fn finish(&mut self) -> ToolParserResult { - Ok(ToolParserOutput::default()) - } - - fn reset(&mut self) -> String { - String::new() - } - } - fn chat_tool(name: &str, strict: Option) -> Tool { Tool { name: name.to_string(), @@ -123,6 +90,10 @@ mod tests { } } + fn qwen3_coder_parser(tools: &[Tool]) -> Box { + Qwen3CoderToolParser::create(tools).expect("Qwen3 Coder parser should build") + } + fn request(tool_choice: ChatToolChoice, tools: Vec) -> ChatRequest { ChatRequest { tool_choice, @@ -147,8 +118,9 @@ mod tests { #[test] fn auto_strict_tool_choice_builds_structural_tag() { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); + let parser = qwen3_coder_parser(&request.tools); - apply_structural_tag_constraint(&mut request, &StructuralTagParser) + apply_structural_tag_constraint(&mut request, parser.as_ref()) .expect("structural tag should build"); let tag = structural_tag_value(&request); @@ -159,8 +131,9 @@ mod tests { #[test] fn auto_non_strict_tool_choice_skips_structural_tag() { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", None)]); + let parser = qwen3_coder_parser(&request.tools); - apply_structural_tag_constraint(&mut request, &StructuralTagParser) + apply_structural_tag_constraint(&mut request, parser.as_ref()) .expect("structural tag decision should succeed"); assert!(request.sampling_params.structured_outputs.is_none()); @@ -169,8 +142,9 @@ mod tests { #[test] fn required_tool_choice_builds_structural_tag_without_strict_tools() { let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); + let parser = qwen3_coder_parser(&request.tools); - apply_structural_tag_constraint(&mut request, &StructuralTagParser) + apply_structural_tag_constraint(&mut request, parser.as_ref()) .expect("structural tag should build"); let tag = structural_tag_value(&request); @@ -186,8 +160,9 @@ mod tests { }, vec![chat_tool("search", None), chat_tool("lookup", None)], ); + let parser = qwen3_coder_parser(&request.tools); - apply_structural_tag_constraint(&mut request, &StructuralTagParser) + apply_structural_tag_constraint(&mut request, parser.as_ref()) .expect("structural tag should build"); let tag = structural_tag_value(&request).to_string(); @@ -198,8 +173,9 @@ mod tests { #[test] fn none_tool_choice_skips_structural_tag() { let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); + let parser = qwen3_coder_parser(&request.tools); - apply_structural_tag_constraint(&mut request, &StructuralTagParser) + apply_structural_tag_constraint(&mut request, parser.as_ref()) .expect("structural tag decision should succeed"); assert!(request.sampling_params.structured_outputs.is_none()); diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index 90b82206ca36..ca68b56223f8 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -1,7 +1,6 @@ use super::types::ChatCompletionRequest; use crate::error::{ApiError, bail_invalid_request}; -use crate::routes::openai::utils::structured_outputs::ResponseFormat; -use crate::routes::openai::utils::types::{ChatMessage, Tool, ToolChoice, ToolChoiceValue}; +use crate::routes::openai::utils::types::{ChatMessage, Tool}; /// Enforce the minimal compatibility contract for the Rust OpenAI server. pub(super) fn validate_request_compat( @@ -59,35 +58,6 @@ pub(super) fn validate_request_compat( } } - if let Some(tool_choice) = &request.tool_choice { - match tool_choice { - ToolChoice::Value( - ToolChoiceValue::Auto | ToolChoiceValue::None | ToolChoiceValue::Required, - ) - | ToolChoice::Function { .. } => {} - ToolChoice::AllowedTools { .. } => { - bail_invalid_request!( - param = "tool_choice", - "allowed_tools tool_choice is not supported yet." - ); - } - } - } - - if has_explicit_structured_output_guidance(request) - && tool_choice_uses_structural_tag_constraint(request) - { - let param = request - .structured_outputs - .as_ref() - .map(|_| "structured_outputs") - .unwrap_or("response_format"); - bail_invalid_request!( - param = param, - "Explicit structured output guidance cannot be combined with strict tool calling." - ); - } - if request.use_beam_search { bail_invalid_request!( param = "use_beam_search", @@ -133,30 +103,6 @@ pub(super) fn validate_request_compat( Ok(()) } -fn has_explicit_structured_output_guidance(request: &ChatCompletionRequest) -> bool { - request.structured_outputs.is_some() - || matches!( - request.response_format.as_ref(), - Some(ResponseFormat::JsonObject | ResponseFormat::JsonSchema { .. }) - ) -} - -fn tool_choice_uses_structural_tag_constraint(request: &ChatCompletionRequest) -> bool { - match &request.tool_choice { - Some(ToolChoice::Value(ToolChoiceValue::Required)) | Some(ToolChoice::Function { .. }) => { - true - } - Some(ToolChoice::Value(ToolChoiceValue::Auto)) => request - .tools - .as_ref() - .is_some_and(|tools| tools.iter().any(|tool| tool.function.strict == Some(true))), - Some(ToolChoice::Value(ToolChoiceValue::None)) | Some(ToolChoice::AllowedTools { .. }) => { - false - } - None => false, - } -} - /// Reject one option unless it is entirely absent. fn reject_non_default( value: Option<&T>, @@ -190,7 +136,7 @@ mod tests { use crate::routes::openai::utils::structured_outputs::ResponseFormat; use crate::routes::openai::utils::types::{ ChatMessage, Function, FunctionChoice, MessageContent, StringOrArray, Tool, ToolChoice, - ToolChoiceValue, ToolReference, + ToolChoiceValue, }; fn served(names: &[&str]) -> Vec { @@ -387,107 +333,4 @@ mod tests { validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) .expect("tool_choice=none is ok"); } - - #[test] - fn validate_request_compat_accepts_required_and_named_tool_choices() { - let tools = Some(vec![Tool { - tool_type: "function".to_string(), - function: Function { - name: "tool".to_string(), - description: None, - parameters: json!({}), - strict: None, - }, - }]); - let required = ChatCompletionRequest { - tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)), - tools: tools.clone(), - ..base_request() - }; - validate_request_compat(&required, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) - .expect("tool_choice=required should be accepted"); - - let named = ChatCompletionRequest { - tool_choice: Some(ToolChoice::Function { - tool_type: "function".to_string(), - function: FunctionChoice { - name: "tool".to_string(), - }, - }), - tools, - ..base_request() - }; - validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) - .expect("named function tool_choice should be accepted"); - } - - #[test] - fn validate_request_compat_rejects_guidance_with_strict_tool_calling() { - let strict_tools = Some(vec![Tool { - tool_type: "function".to_string(), - function: Function { - name: "tool".to_string(), - description: None, - parameters: json!({}), - strict: Some(true), - }, - }]); - let named = ChatCompletionRequest { - response_format: Some(ResponseFormat::JsonObject), - tool_choice: Some(ToolChoice::Function { - tool_type: "function".to_string(), - function: FunctionChoice { - name: "tool".to_string(), - }, - }), - tools: strict_tools.clone(), - ..base_request() - }; - assert!(validate_request_compat(&named, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); - - let auto = ChatCompletionRequest { - structured_outputs: Some(json!({"json": {"type": "object"}})), - tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Auto)), - tools: strict_tools, - ..base_request() - }; - assert!(validate_request_compat(&auto, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err()); - } - - #[test] - fn validate_request_compat_allows_text_response_format_with_strict_tool_calling() { - let request = ChatCompletionRequest { - response_format: Some(ResponseFormat::Text), - tool_choice: Some(ToolChoice::Value(ToolChoiceValue::Required)), - tools: Some(vec![Tool { - tool_type: "function".to_string(), - function: Function { - name: "tool".to_string(), - description: None, - parameters: json!({}), - strict: Some(true), - }, - }]), - ..base_request() - }; - validate_request_compat(&request, &served(&["Qwen/Qwen1.5-0.5B-Chat"])) - .expect("response_format=text should not conflict with strict tool calling"); - } - - #[test] - fn validate_request_compat_rejects_allowed_tools_choice() { - let allowed_tools = ChatCompletionRequest { - tool_choice: Some(ToolChoice::AllowedTools { - tool_type: "allowed_tools".to_string(), - mode: "auto".to_string(), - tools: vec![ToolReference::Function { - name: "tool".to_string(), - }], - }), - ..base_request() - }; - assert!( - validate_request_compat(&allowed_tools, &served(&["Qwen/Qwen1.5-0.5B-Chat"])).is_err() - ); - } } diff --git a/rust/src/server/src/routes/openai/utils/structured_outputs.rs b/rust/src/server/src/routes/openai/utils/structured_outputs.rs index 12b265cf0d55..e974c836bb5e 100644 --- a/rust/src/server/src/routes/openai/utils/structured_outputs.rs +++ b/rust/src/server/src/routes/openai/utils/structured_outputs.rs @@ -53,12 +53,6 @@ pub enum ResponseFormat { fn deserialize_structured_outputs( raw: &serde_json::Value, ) -> Result { - if raw.get("_backend").is_some() { - return Err(ApiError::invalid_request( - "request-level structured output backend selection is not supported", - Some("structured_outputs._backend"), - )); - } serde_json::from_value(raw.clone()).map_err(|e| { ApiError::invalid_request( format!("invalid structured_outputs: {e}"), @@ -136,32 +130,3 @@ pub fn convert_from_response_format_value( })?; convert_from_response_format(Some(&fmt), &None) } - -#[cfg(test)] -mod tests { - use serde_json::json; - - use super::convert_from_response_format; - - #[test] - fn rejects_request_level_structured_output_backend() { - let err = convert_from_response_format( - None, - &Some(json!({ - "json_object": true, - "_backend": "guidance", - })), - ) - .expect_err("request-level backend selection should be rejected"); - - let response = err.to_error_response(); - assert_eq!( - response.error.message, - "request-level structured output backend selection is not supported" - ); - assert_eq!( - response.error.param.as_deref(), - Some("structured_outputs._backend") - ); - } -} From fd790673e794932fba4e45eff15bb64d2cb8438e Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Mon, 22 Jun 2026 19:47:40 +0800 Subject: [PATCH 4/6] add more docs Signed-off-by: Bugen Zhao --- .../chat/src/output/default/structural_tag.rs | 100 +++++++++++++++--- .../openai/chat_completions/validate.rs | 3 +- 2 files changed, 88 insertions(+), 15 deletions(-) diff --git a/rust/src/chat/src/output/default/structural_tag.rs b/rust/src/chat/src/output/default/structural_tag.rs index 51013e6c6c41..eb1fa3d14362 100644 --- a/rust/src/chat/src/output/default/structural_tag.rs +++ b/rust/src/chat/src/output/default/structural_tag.rs @@ -4,13 +4,15 @@ use thiserror_ext::AsReport; use vllm_engine_core_client::protocol::{StructuredOutputBackend, StructuredOutputsParams}; use xgrammar_structural_tag::{ FunctionDefinition, FunctionToolParam, ToolChoice as StructuralTagToolChoice, ToolParam, - build_optional_structural_tag, + build_structural_tag, }; use crate::parser::tool::ToolParser; use crate::request::{ChatRequest, ChatToolChoice}; use crate::{Error, Result as ChatResult}; +/// Apply structural tag constraints to the request based on the tool parser's structural tag +/// support and the request's tool choice. pub(super) fn apply_structural_tag_constraint( request: &mut ChatRequest, parser: &dyn ToolParser, @@ -35,33 +37,37 @@ pub(super) fn apply_structural_tag_constraint( }) .collect::>(); - let tag = - build_optional_structural_tag(model, &tools, tool_choice, false).map_err(|error| { - Error::StructuralTag { - message: error.to_report_string(), - } + let structural_tag = build_structural_tag(model, &tools, tool_choice, false) + .and_then(|tag| tag.to_json_string()) + .map_err(|error| Error::StructuralTag { + message: error.to_report_string(), })?; - let Some(tag) = tag else { - return Ok(()); - }; - let structural_tag = tag.to_json_string().map_err(|error| Error::StructuralTag { - message: error.to_report_string(), - })?; + // Overwrite any existing structured output settings with the structural tag constraint. request.sampling_params.structured_outputs = Some(StructuredOutputsParams { structural_tag: Some(structural_tag), backend: StructuredOutputBackend::Xgrammar, ..Default::default() }); + Ok(()) } +/// Resolve the tool choice used for [`xgrammar_structural_tag`] based on the request. +/// +/// Returns `None` if no structural tag constraints should be applied. fn structural_tag_tool_choice(request: &ChatRequest) -> Option { + if request.tools.is_empty() { + return None; + } + match &request.tool_choice { + // For `Auto`, only apply the structural tag if there's at least one strict tool. ChatToolChoice::Auto if request.tools.iter().any(|tool| tool.strict == Some(true)) => { Some(StructuralTagToolChoice::auto()) } ChatToolChoice::Auto | ChatToolChoice::None => None, + ChatToolChoice::Required => Some(StructuralTagToolChoice::required()), ChatToolChoice::Function { name } => Some(StructuralTagToolChoice::function(name.clone())), } @@ -70,7 +76,7 @@ fn structural_tag_tool_choice(request: &ChatRequest) -> Option &StructuredOutputsParams { + request + .sampling_params + .structured_outputs + .as_ref() + .expect("structured outputs should be set") + } + #[test] fn auto_strict_tool_choice_builds_structural_tag() { let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); @@ -139,6 +153,27 @@ mod tests { assert!(request.sampling_params.structured_outputs.is_none()); } + #[test] + fn auto_strict_tool_choice_overwrites_existing_json_guidance() { + let mut request = request(ChatToolChoice::Auto, vec![chat_tool("search", Some(true))]); + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + json: Some(json!({"type": "object"})), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag should build"); + + let params = structured_outputs(&request); + assert!(params.json.is_none()); + assert!(params.structural_tag.is_some()); + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + #[test] fn required_tool_choice_builds_structural_tag_without_strict_tools() { let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); @@ -152,6 +187,27 @@ mod tests { assert!(tag.to_string().contains("search")); } + #[test] + fn required_tool_choice_overwrites_existing_json_object_guidance() { + let mut request = request(ChatToolChoice::Required, vec![chat_tool("search", None)]); + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + json_object: Some(true), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag should build"); + + let params = structured_outputs(&request); + assert!(params.json_object.is_none()); + assert!(params.structural_tag.is_some()); + let tag = structural_tag_value(&request); + assert_eq!(tag["type"], "structural_tag"); + assert!(tag.to_string().contains("search")); + } + #[test] fn named_tool_choice_builds_structural_tag_for_named_tool_only() { let mut request = request( @@ -180,4 +236,22 @@ mod tests { assert!(request.sampling_params.structured_outputs.is_none()); } + + #[test] + fn none_tool_choice_preserves_existing_json_object_guidance() { + let mut request = request(ChatToolChoice::None, vec![chat_tool("search", Some(true))]); + request.sampling_params.structured_outputs = Some(StructuredOutputsParams { + json_object: Some(true), + backend: StructuredOutputBackend::Xgrammar, + ..Default::default() + }); + let parser = qwen3_coder_parser(&request.tools); + + apply_structural_tag_constraint(&mut request, parser.as_ref()) + .expect("structural tag decision should succeed"); + + let params = structured_outputs(&request); + assert_eq!(params.json_object, Some(true)); + assert!(params.structural_tag.is_none()); + } } diff --git a/rust/src/server/src/routes/openai/chat_completions/validate.rs b/rust/src/server/src/routes/openai/chat_completions/validate.rs index ca68b56223f8..f731cb2cde18 100644 --- a/rust/src/server/src/routes/openai/chat_completions/validate.rs +++ b/rust/src/server/src/routes/openai/chat_completions/validate.rs @@ -135,8 +135,7 @@ mod tests { use crate::routes::openai::chat_completions::types::ChatCompletionRequest; use crate::routes::openai::utils::structured_outputs::ResponseFormat; use crate::routes::openai::utils::types::{ - ChatMessage, Function, FunctionChoice, MessageContent, StringOrArray, Tool, ToolChoice, - ToolChoiceValue, + ChatMessage, Function, MessageContent, StringOrArray, Tool, ToolChoice, ToolChoiceValue, }; fn served(names: &[&str]) -> Vec { From 1b410d305e015692e0d8be1f68ec98065387e46f Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Mon, 22 Jun 2026 19:55:08 +0800 Subject: [PATCH 5/6] bump dep version Signed-off-by: Bugen Zhao --- rust/Cargo.lock | 2 +- rust/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 33f701d4a1a7..372406cbae32 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -6586,7 +6586,7 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xgrammar-structural-tag" version = "0.1.0+xgrammar.0.2.2.4d145cc" -source = "git+https://github.com/Inferact/xgrammar-structural-tag.git?rev=afffd968ecf3ec9ec0963953741dfdf6a00b4e10#afffd968ecf3ec9ec0963953741dfdf6a00b4e10" +source = "git+https://github.com/Inferact/xgrammar-structural-tag.git?rev=8c74b8c415834170364f075bd04457955639744d#8c74b8c415834170364f075bd04457955639744d" dependencies = [ "serde", "serde_json", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 7346627abce5..f9962a6a152c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -123,7 +123,7 @@ vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } vllm-tool-parser = { path = "src/tool-parser" } winnow = "1.0.2" -xgrammar-structural-tag = { git = "https://github.com/Inferact/xgrammar-structural-tag.git", rev = "afffd968ecf3ec9ec0963953741dfdf6a00b4e10" } +xgrammar-structural-tag = { git = "https://github.com/Inferact/xgrammar-structural-tag.git", rev = "8c74b8c415834170364f075bd04457955639744d" } zeromq = { version = "0.6.0", default-features = false, features = [ "tokio-runtime", "all-transport", From 571743a7ad9f47485bb2e5951312ca77214f5723 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 23 Jun 2026 14:37:01 +0800 Subject: [PATCH 6/6] use crates.io version Signed-off-by: Bugen Zhao --- rust/Cargo.lock | 15 ++++++++++++++- rust/Cargo.toml | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 372406cbae32..40d0cd0016dd 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -316,6 +316,17 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "auto_impl" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdcb70bdbc4d478427380519163274ac86e52916e10f0a8889adf0f96d3fee7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -6586,8 +6597,10 @@ checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "xgrammar-structural-tag" version = "0.1.0+xgrammar.0.2.2.4d145cc" -source = "git+https://github.com/Inferact/xgrammar-structural-tag.git?rev=8c74b8c415834170364f075bd04457955639744d#8c74b8c415834170364f075bd04457955639744d" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2436dea2393d55a3b188588aa300c5a8afe8f45a77da52c611fb4498a6c876e6" dependencies = [ + "auto_impl", "serde", "serde_json", "strum", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index f9962a6a152c..e31bf07bbd47 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -123,7 +123,7 @@ vllm-text = { path = "src/text" } vllm-tokenizer = { path = "src/tokenizer" } vllm-tool-parser = { path = "src/tool-parser" } winnow = "1.0.2" -xgrammar-structural-tag = { git = "https://github.com/Inferact/xgrammar-structural-tag.git", rev = "8c74b8c415834170364f075bd04457955639744d" } +xgrammar-structural-tag = "0.1.0" zeromq = { version = "0.6.0", default-features = false, features = [ "tokio-runtime", "all-transport",