From cbc7c7caccd47819c46ed7b981838f9fda85a0e9 Mon Sep 17 00:00:00 2001 From: yifeng liu <31553858+pallasathena92@users.noreply.github.com> Date: Thu, 13 Aug 2026 05:27:34 -0700 Subject: [PATCH 1/2] fix(tools): honor parallel_tool_calls=false in the required-tools grammar The JSON-schema fallback constraint for tool_choice=required emits an array schema with minItems 1 and no upper bound, and nothing in the constraint path consults the request's parallel-tool-calls setting - so parallel_tool_calls:false (or Anthropic disable_parallel_tool_use:true) still lets guided decoding produce several calls. The constraint generator now takes the request's parallel-tool-calls setting directly and stamps maxItems 1 on the required-tools array schema when parallel calls are disabled. All five call sites - both gRPC preparation stages and the three Go-binding entry points - pass their request's real value, which fixes the same gap on the Go SDK surface. Named-function constraints are already single-call by construction; structural-tag grammars carry no repeat bound (documented limitation). Signed-off-by: yifeng liu <31553858+pallasathena92@users.noreply.github.com> --- bindings/golang/src/client.rs | 7 +- bindings/golang/src/policy.rs | 7 +- bindings/golang/src/preprocessor.rs | 7 +- crates/tool_parser/src/factory.rs | 21 +++++- crates/tool_parser/src/tests.rs | 72 +++++++++++++++++++ .../grpc/regular/stages/chat/preparation.rs | 1 + .../regular/stages/messages/preparation.rs | 18 ++++- 7 files changed, 126 insertions(+), 7 deletions(-) diff --git a/bindings/golang/src/client.rs b/bindings/golang/src/client.rs index 13615ff7b0..2342510a54 100644 --- a/bindings/golang/src/client.rs +++ b/bindings/golang/src/client.rs @@ -183,7 +183,12 @@ pub unsafe extern "C" fn sgl_client_chat_completion_stream( chat_request.tools.as_ref(), chat_request.tool_choice.as_ref(), ) { - match registry.generate_tool_constraint(None, tools, tool_choice) { + match registry.generate_tool_constraint( + None, + tools, + tool_choice, + chat_request.parallel_tool_calls, + ) { Ok(Some(c)) => Some(c.to_tuple()), Ok(None) => None, Err(e) => { diff --git a/bindings/golang/src/policy.rs b/bindings/golang/src/policy.rs index 4a6f37ddf1..2d13c7c7f2 100644 --- a/bindings/golang/src/policy.rs +++ b/bindings/golang/src/policy.rs @@ -609,7 +609,12 @@ pub unsafe extern "C" fn sgl_multi_client_chat_completion_stream( chat_request.tools.as_ref(), chat_request.tool_choice.as_ref(), ) { - match registry.generate_tool_constraint(None, tools, tool_choice) { + match registry.generate_tool_constraint( + None, + tools, + tool_choice, + chat_request.parallel_tool_calls, + ) { Ok(Some(c)) => Some(c.to_tuple()), Ok(None) => None, Err(e) => { diff --git a/bindings/golang/src/preprocessor.rs b/bindings/golang/src/preprocessor.rs index 2e4cd0ca39..4743cf34fe 100644 --- a/bindings/golang/src/preprocessor.rs +++ b/bindings/golang/src/preprocessor.rs @@ -64,7 +64,12 @@ fn preprocess_impl( chat_request.tools.as_ref(), chat_request.tool_choice.as_ref(), ) { - match registry.generate_tool_constraint(None, tools, tool_choice) { + match registry.generate_tool_constraint( + None, + tools, + tool_choice, + chat_request.parallel_tool_calls, + ) { Ok(Some(c)) => { let json_str = serde_json::to_string(&c.to_tuple()).map_err(|e| { ( diff --git a/crates/tool_parser/src/factory.rs b/crates/tool_parser/src/factory.rs index 0e688db53f..767da65c16 100644 --- a/crates/tool_parser/src/factory.rs +++ b/crates/tool_parser/src/factory.rs @@ -188,12 +188,20 @@ impl ParserRegistry { /// If `configured_parser` supports structural tags → `StructuralTag(json)`. /// Otherwise → `JsonSchema(schema)` for required/function tool_choice. /// Returns `Ok(None)` for auto/none tool_choice. + /// `parallel_tool_calls` is the request's setting (OpenAI + /// `parallel_tool_calls`, or the inverse of Anthropic + /// `disable_parallel_tool_use`); `Some(false)` bounds the JSON-schema + /// fallback to a single call. Named-function constraints are single-call + /// by construction; structural tags are unaffected (tag grammars carry no + /// repeat bound). pub fn generate_tool_constraint( &self, configured_parser: Option<&str>, tools: &[Tool], tool_choice: &ToolChoice, + parallel_tool_calls: Option, ) -> Result, String> { + let single_tool_call = parallel_tool_calls == Some(false); if tools.is_empty() { return Ok(None); } @@ -226,7 +234,7 @@ impl ParserRegistry { Ok(Some(ToolConstraint::JsonSchema(params_schema))) } _ => { - let schema = build_required_array_schema(tools)?; + let schema = build_required_array_schema(tools, single_tool_call)?; Ok(Some(ToolConstraint::JsonSchema(schema))) } } @@ -539,8 +547,9 @@ impl Default for ParserFactory { } } -/// Build JSON schema for required tool calls (array with minItems: 1). -fn build_required_array_schema(tools: &[Tool]) -> Result { +/// Build JSON schema for required tool calls (array with minItems: 1; also +/// maxItems: 1 when the request disables parallel tool calls). +fn build_required_array_schema(tools: &[Tool], single_call: bool) -> Result { let mut any_of_schemas = Vec::with_capacity(tools.len()); for tool in tools { let tool_schema = json!({ @@ -585,6 +594,12 @@ fn build_required_array_schema(tools: &[Tool]) -> Result { } }); + if single_call { + if let serde_json::Value::Object(ref mut obj) = array_schema { + obj.insert("maxItems".to_string(), json!(1)); + } + } + if !all_defs.is_empty() { if let serde_json::Value::Object(ref mut obj) = array_schema { obj.insert("$defs".to_string(), serde_json::Value::Object(all_defs)); diff --git a/crates/tool_parser/src/tests.rs b/crates/tool_parser/src/tests.rs index d3b1326876..a021a4ffc9 100644 --- a/crates/tool_parser/src/tests.rs +++ b/crates/tool_parser/src/tests.rs @@ -818,3 +818,75 @@ mod minimax_mapping_tests { } } } + +mod constraint_limit_tests { + use openai_protocol::common::{Function, FunctionChoice, Tool, ToolChoice, ToolChoiceValue}; + + use crate::{factory::ParserFactory, ToolConstraint}; + + fn tools() -> Vec { + vec![Tool { + tool_type: "function".to_string(), + function: Function { + name: "get_weather".to_string(), + description: Some("Get weather information".to_string()), + parameters: serde_json::json!({ + "type": "object", + "properties": { + "city": {"type": "string"} + } + }), + strict: None, + }, + }] + } + + fn schema_for(choice: &ToolChoice, parallel_tool_calls: Option) -> serde_json::Value { + let factory = ParserFactory::new(); + let constraint = factory + .registry() + .generate_tool_constraint(None, &tools(), choice, parallel_tool_calls) + .unwrap() + .expect("constraint expected"); + match constraint { + ToolConstraint::JsonSchema(s) => serde_json::from_str(&s).unwrap(), + ToolConstraint::StructuralTag(_) => panic!("expected JSON schema constraint"), + } + } + + #[test] + fn required_bounds_to_single_call_when_parallel_disabled() { + let schema = schema_for(&ToolChoice::Value(ToolChoiceValue::Required), Some(false)); + assert_eq!(schema["minItems"], 1); + assert_eq!(schema["maxItems"], 1); + } + + #[test] + fn required_stays_unbounded_unless_parallel_disabled() { + // Unspecified and explicitly-true behave identically: no upper bound. + for parallel in [None, Some(true)] { + let schema = schema_for(&ToolChoice::Value(ToolChoiceValue::Required), parallel); + assert_eq!(schema["minItems"], 1); + assert!( + schema.get("maxItems").is_none(), + "schema must not carry maxItems for parallel_tool_calls={parallel:?}" + ); + } + } + + #[test] + fn named_choice_is_unaffected_by_parallel_setting() { + let named = ToolChoice::Function { + tool_type: "function".to_string(), + function: FunctionChoice { + name: "get_weather".to_string(), + }, + }; + let bounded = schema_for(&named, Some(false)); + let unbounded = schema_for(&named, None); + // Named choice constrains to the single tool's parameters object — + // single-call by construction, so the setting must be a no-op. + assert_eq!(bounded, unbounded); + assert_eq!(bounded["type"], "object"); + } +} diff --git a/model_gateway/src/routers/grpc/regular/stages/chat/preparation.rs b/model_gateway/src/routers/grpc/regular/stages/chat/preparation.rs index dad3a4d01a..821ad121df 100644 --- a/model_gateway/src/routers/grpc/regular/stages/chat/preparation.rs +++ b/model_gateway/src/routers/grpc/regular/stages/chat/preparation.rs @@ -269,6 +269,7 @@ pub(crate) async fn prepare_chat_like( .as_deref(), tools, tool_choice, + request.parallel_tool_calls, ) .map_err(|e| { error!(function = "ChatPreparationStage::execute", error = %e, "Invalid tool configuration"); diff --git a/model_gateway/src/routers/grpc/regular/stages/messages/preparation.rs b/model_gateway/src/routers/grpc/regular/stages/messages/preparation.rs index b72fe93f1d..2c5439a7e0 100644 --- a/model_gateway/src/routers/grpc/regular/stages/messages/preparation.rs +++ b/model_gateway/src/routers/grpc/regular/stages/messages/preparation.rs @@ -6,7 +6,7 @@ use async_trait::async_trait; use axum::response::Response; use openai_protocol::{ common::{StringOrArray, ToolChoice, ToolChoiceValue}, - messages::CreateMessageRequest, + messages::{CreateMessageRequest, ToolChoice as MessagesToolChoice}, }; use tracing::{debug, error}; @@ -259,6 +259,21 @@ impl MessagePreparationStage { } } + // Anthropic spells the parallelism setting inversely on tool_choice. + let parallel_tool_calls = request.tool_choice.as_ref().and_then(|tc| match tc { + MessagesToolChoice::Auto { + disable_parallel_tool_use, + } + | MessagesToolChoice::Any { + disable_parallel_tool_use, + } + | MessagesToolChoice::Tool { + disable_parallel_tool_use, + .. + } => disable_parallel_tool_use.map(|disable| !disable), + MessagesToolChoice::None => None, + }); + // Step 4: Build tool constraints if tools present let tool_call_constraint = if let (false, Some(tool_choice)) = (filtered_tools.is_empty(), chat_tool_choice.as_ref()) @@ -273,6 +288,7 @@ impl MessagePreparationStage { .as_deref(), &filtered_tools, tool_choice, + parallel_tool_calls, ) .map_err(|e| { error!(function = "MessagePreparationStage::execute", error = %e, "Invalid tool configuration"); From 3a69e5bcb4be3d328bdd6affbdbb2dbe1ca141c3 Mon Sep 17 00:00:00 2001 From: Alex McC <319643551+hello-alexmcc@users.noreply.github.com> Date: Sat, 5 Sep 2026 10:10:33 -0700 Subject: [PATCH 2/2] fix(tools): bound the Harmony required-tools grammar under parallel_tool_calls=false The Harmony preparation stage builds its own structural tag and never consulted the request's parallel-tool-calls setting, so tool_choice required with parallel_tool_calls:false still admitted repeated calls on that path. Each Harmony tag is exactly one call, so stop_after_first is the repeat bound: it is now set for a named function and whenever the request disables parallel calls, on both the chat and the responses call sites. Also rewords the constraint doc: structural-tag formats can express a repeat bound; the registry builders just do not set one yet. Signed-off-by: Alex McC <319643551+hello-alexmcc@users.noreply.github.com> --- crates/tool_parser/src/factory.rs | 6 +- .../grpc/harmony/stages/preparation.rs | 132 ++++++++++++++++-- 2 files changed, 128 insertions(+), 10 deletions(-) diff --git a/crates/tool_parser/src/factory.rs b/crates/tool_parser/src/factory.rs index 767da65c16..a2b158ebfb 100644 --- a/crates/tool_parser/src/factory.rs +++ b/crates/tool_parser/src/factory.rs @@ -192,8 +192,10 @@ impl ParserRegistry { /// `parallel_tool_calls`, or the inverse of Anthropic /// `disable_parallel_tool_use`); `Some(false)` bounds the JSON-schema /// fallback to a single call. Named-function constraints are single-call - /// by construction; structural tags are unaffected (tag grammars carry no - /// repeat bound). + /// by construction. Registry structural-tag builders do not currently set + /// a repeat bound (`stop_after_first`), so the setting is unenforced on + /// that path; the Harmony preparation stage builds its own tag and does + /// honor it. pub fn generate_tool_constraint( &self, configured_parser: Option<&str>, diff --git a/model_gateway/src/routers/grpc/harmony/stages/preparation.rs b/model_gateway/src/routers/grpc/harmony/stages/preparation.rs index 02f066a929..b89f7b8001 100644 --- a/model_gateway/src/routers/grpc/harmony/stages/preparation.rs +++ b/model_gateway/src/routers/grpc/harmony/stages/preparation.rs @@ -99,8 +99,12 @@ impl HarmonyPreparationStage { // Step 2: Build structural tag constraint let tool_constraint = if let Some(tools) = body_ref.tools.as_ref() { - Self::generate_tool_call_constraint(tools, body_ref.tool_choice.as_ref()) - .map_err(|e| *e)? + Self::generate_tool_call_constraint( + tools, + body_ref.tool_choice.as_ref(), + body_ref.parallel_tool_calls, + ) + .map_err(|e| *e)? } else { None }; @@ -183,8 +187,12 @@ impl HarmonyPreparationStage { let tool_constraint = if function_tools.is_empty() { None } else { - Self::generate_tool_call_constraint(&function_tools, chat_tool_choice.as_ref()) - .map_err(|e| *e)? + Self::generate_tool_call_constraint( + &function_tools, + chat_tool_choice.as_ref(), + request.parallel_tool_calls, + ) + .map_err(|e| *e)? }; let text_constraint = if let Some(text_config) = &request.text { @@ -303,26 +311,36 @@ impl HarmonyPreparationStage { /// /// Uses structural tags with `triggered_tags` format to force Harmony format output. /// This ensures the model outputs in Harmony format (with channels) even when constrained. + /// + /// `parallel_tool_calls` is the request's setting; `Some(false)` bounds a + /// `required`-style constraint to a single call. Named-function constraints + /// are single-call regardless. fn generate_tool_call_constraint( tools: &[Tool], tool_choice: Option<&ToolChoice>, + parallel_tool_calls: Option, ) -> Result, Box> { let Some(choice) = tool_choice else { return Ok(None); }; + let single_tool_call = parallel_tool_calls == Some(false); match choice { ToolChoice::Function { function, .. } => { - let tag = Self::build_tool_call_structural_tag(tools, Some(&function.name))?; + let tag = Self::build_tool_call_structural_tag( + tools, + Some(&function.name), + single_tool_call, + )?; Ok(Some(("structural_tag".to_string(), tag))) } ToolChoice::Value(ToolChoiceValue::Required) => { - let tag = Self::build_tool_call_structural_tag(tools, None)?; + let tag = Self::build_tool_call_structural_tag(tools, None, single_tool_call)?; Ok(Some(("structural_tag".to_string(), tag))) } ToolChoice::AllowedTools { mode, .. } => { if mode == "required" { - let tag = Self::build_tool_call_structural_tag(tools, None)?; + let tag = Self::build_tool_call_structural_tag(tools, None, single_tool_call)?; Ok(Some(("structural_tag".to_string(), tag))) } else { Ok(None) @@ -337,9 +355,15 @@ impl HarmonyPreparationStage { /// Supports both reasoning-enabled and reasoning-disabled modes: /// - With reasoning: triggers on `<|start|>assistant<|channel|>commentary` (waits for analysis) /// - Without reasoning: triggers on `<|channel|>commentary` (goes directly to commentary) + /// + /// `single_tool_call` requests a repeat bound: the grammar stops accepting + /// further tool-call tags after the first one completes. A named function + /// is always single-call; a `required` constraint is single-call only when + /// the request disabled parallel tool calls. fn build_tool_call_structural_tag( tools: &[Tool], specific_function: Option<&str>, + single_tool_call: bool, ) -> Result> { let mut tags = Vec::new(); @@ -395,7 +419,9 @@ impl HarmonyPreparationStage { })); } - let stop_after_first = specific_function.is_some(); + // Each tag is exactly one call, so `stop_after_first` is the repeat + // bound: set for a named function and for parallel_tool_calls=false. + let stop_after_first = specific_function.is_some() || single_tool_call; let structural_tag = json!({ "format": { @@ -464,3 +490,93 @@ pub(crate) fn build_text_format_structural_tag( serde_json::to_string(&structural_tag) .map_err(|e| format!("Failed to serialize structural tag for structured output: {e}")) } + +#[cfg(test)] +mod tests { + use openai_protocol::common::{Function, FunctionChoice, Tool, ToolChoice, ToolChoiceValue}; + + use super::HarmonyPreparationStage; + + fn tools() -> Vec { + ["get_weather", "get_time"] + .into_iter() + .map(|name| Tool { + tool_type: "function".to_string(), + function: Function { + name: name.to_string(), + description: None, + parameters: serde_json::json!({"type": "object", "properties": {}}), + strict: None, + }, + }) + .collect() + } + + fn stop_after_first( + tool_choice: &ToolChoice, + parallel_tool_calls: Option, + ) -> (bool, usize) { + let (_, tag) = HarmonyPreparationStage::generate_tool_call_constraint( + &tools(), + Some(tool_choice), + parallel_tool_calls, + ) + .unwrap() + .expect("constraint expected"); + let tag: serde_json::Value = serde_json::from_str(&tag).unwrap(); + let format = &tag["format"]; + ( + format["stop_after_first"].as_bool().unwrap(), + format["tags"].as_array().unwrap().len(), + ) + } + + fn required() -> ToolChoice { + ToolChoice::Value(ToolChoiceValue::Required) + } + + #[test] + fn required_bounds_to_single_call_when_parallel_disabled() { + let (stop, tag_count) = stop_after_first(&required(), Some(false)); + assert!(stop, "parallel_tool_calls=false must set the repeat bound"); + // Both tools stay eligible; only the repeat count is bounded. + assert_eq!(tag_count, 4); + } + + #[test] + fn required_stays_unbounded_unless_parallel_disabled() { + for parallel in [None, Some(true)] { + let (stop, _) = stop_after_first(&required(), parallel); + assert!( + !stop, + "required must not carry a repeat bound for parallel_tool_calls={parallel:?}" + ); + } + } + + #[test] + fn allowed_tools_required_mode_honors_parallel_setting() { + let choice = ToolChoice::AllowedTools { + tool_type: "allowed_tools".to_string(), + mode: "required".to_string(), + tools: vec![], + }; + assert!(stop_after_first(&choice, Some(false)).0); + assert!(!stop_after_first(&choice, None).0); + } + + #[test] + fn named_function_is_single_call_regardless_of_parallel_setting() { + let named = ToolChoice::Function { + tool_type: "function".to_string(), + function: FunctionChoice { + name: "get_weather".to_string(), + }, + }; + for parallel in [None, Some(true), Some(false)] { + let (stop, tag_count) = stop_after_first(&named, parallel); + assert!(stop, "named function must always be single-call"); + assert_eq!(tag_count, 2, "named choice keeps only the named tool"); + } + } +}