diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index f06ea4b4b40a..8971e7896571 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -3244,6 +3244,11 @@ async fn responses( return Ok(resp.into_response()); } + if let Err(err_response) = validate_response_chat_template_args(&request) { + inflight_guard.mark_error(extract_error_type_from_response(&err_response)); + return Err(err_response); + } + // Validate sampling and output parameters if let Err(err_response) = validate_responses_fields(&request) { inflight_guard.mark_error(extract_error_type_from_response(&err_response)); @@ -3344,6 +3349,13 @@ async fn responses( ), ); + let move_reasoning_to_content_when_empty = + crate::preprocessor::OpenAIPreprocessor::wants_reasoning_as_content_when_empty( + request.chat_template_args.as_ref(), + ); + let parsing_options = parsing_options + .with_move_reasoning_to_content_when_empty(move_reasoning_to_content_when_empty); + // Responses requests share the chat-completions aggregator for the unary // path. Thread this option through so its post-parse fallback also caps a // model-produced batch to the first tool call when parallel calls are @@ -3351,21 +3363,6 @@ async fn responses( let parsing_options = parsing_options.with_parallel_tool_calls(request.inner.parallel_tool_calls); - // NOTE: `move_reasoning_to_content_when_empty` is the aggregator flag and is - // not set here. A non-streaming Responses request DOES reach the aggregator - // (forcing stream=true on the converted request only drives internal - // streaming; the client-facing `streaming` flag still selects the aggregating - // branch below), so it reaches it with the flag false. - // - // That is currently unreachable rather than wrong: the Responses-to-chat - // conversion hard-codes `chat_template_args: None`, so - // `force_nonempty_content` can never be set on this path in the first place. - // If that conversion ever forwards chat_template_args, the streaming stage - // would still cover the reasoning-only case — `postprocessor_parsing_stream` - // gates on the request's own args via `wants_reasoning_as_content_when_empty` - // rather than on this flag — but the aggregator backstop should be wired here - // too at that point. Tracked as follow-up. - let mut response_collector = state .metrics_clone() .create_response_collector(&metric_model); @@ -3589,6 +3586,20 @@ pub fn validate_response_unsupported_fields( None } +fn validate_response_chat_template_args( + request: &NvCreateResponse, +) -> Result<(), ErrorResponse> { + crate::protocols::openai::validate::validate_chat_template_args( + request.chat_template_args.as_ref(), + ) + .map_err(|e| { + ErrorMessage::from_http_error(HttpError { + code: 400, + message: VALIDATION_PREFIX.to_string() + &e.to_string(), + }) + }) +} + /// Validates sampling and output parameters on the Responses API request. pub fn validate_responses_fields(request: &NvCreateResponse) -> Result<(), ErrorResponse> { use crate::protocols::openai::validate; @@ -4988,6 +4999,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, } } @@ -5573,6 +5585,30 @@ mod tests { assert!(validate_response_unsupported_fields(&request).is_some()); } + #[test] + fn test_validate_response_chat_template_args_rejects_nested_chat_template() { + let mut request = make_base_request(); + request.chat_template_args = Some(HashMap::from([( + "chat_template".to_string(), + serde_json::json!("{{ 'pwned' }}"), + )])); + + let (status, _) = validate_response_chat_template_args(&request).unwrap_err(); + assert_eq!(status, StatusCode::BAD_REQUEST); + } + + #[test] + fn test_validate_response_chat_template_args_accepts_ordinary_args() { + let mut request = make_base_request(); + request.chat_template_args = Some(HashMap::from([( + "enable_thinking".to_string(), + serde_json::json!(true), + )])); + + assert!(validate_response_chat_template_args(&request).is_ok()); + assert!(validate_response_chat_template_args(&make_base_request()).is_ok()); + } + #[test] fn test_validate_unsupported_fields_accepts_supported_nvext_fields() { let mut request = make_base_request(); diff --git a/lib/llm/src/protocols/common/input_trigger.rs b/lib/llm/src/protocols/common/input_trigger.rs index ff340f86c48b..8165a4c1413c 100644 --- a/lib/llm/src/protocols/common/input_trigger.rs +++ b/lib/llm/src/protocols/common/input_trigger.rs @@ -185,6 +185,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, } } diff --git a/lib/llm/src/protocols/openai/responses/mod.rs b/lib/llm/src/protocols/openai/responses/mod.rs index 1d2770465740..c7317ba72ff9 100644 --- a/lib/llm/src/protocols/openai/responses/mod.rs +++ b/lib/llm/src/protocols/openai/responses/mod.rs @@ -51,6 +51,13 @@ pub struct NvCreateResponse { #[serde(skip_serializing_if = "Option::is_none")] #[schema(value_type = Object)] pub nvext: Option, + + #[serde( + default, + skip_serializing_if = "Option::is_none", + alias = "chat_template_kwargs" + )] + pub chat_template_args: Option>, } #[derive(ToSchema, Deserialize, Validate, Debug, Clone)] @@ -891,7 +898,7 @@ impl TryFrom for NvCreateChatCompletionRequest { }, common: Default::default(), nvext: resp.nvext, - chat_template_args: None, + chat_template_args: resp.chat_template_args, thinking: None, media_io_kwargs: None, return_tokens_as_token_ids: None, @@ -1348,6 +1355,7 @@ mod tests { annotations: Some(vec!["debug".into(), "trace".into()]), ..Default::default() }), + chat_template_args: None, } } @@ -1424,6 +1432,64 @@ mod tests { assert_eq!(nv_req.inner.store, Some(true)); } + #[test] + fn test_responses_chat_template_args_forwarded_to_chat_completion() { + let req: NvCreateResponse = serde_json::from_value(serde_json::json!({ + "model": "dummy-model", + "input": "hello", + "chat_template_args": { "enable_thinking": true } + })) + .unwrap(); + + let chat_req = NvCreateChatCompletionRequest::try_from(req).unwrap(); + + let args = chat_req + .chat_template_args + .expect("chat_template_args must survive the Responses to Chat conversion"); + assert_eq!(args.get("enable_thinking"), Some(&serde_json::json!(true))); + } + + #[test] + fn test_responses_chat_template_kwargs_alias_forwarded_to_chat_completion() { + let req: NvCreateResponse = serde_json::from_value(serde_json::json!({ + "model": "dummy-model", + "input": "hello", + "chat_template_kwargs": { "enable_thinking": true } + })) + .unwrap(); + + let chat_req = NvCreateChatCompletionRequest::try_from(req).unwrap(); + + let args = chat_req + .chat_template_args + .expect("chat_template_kwargs alias must bind chat_template_args"); + assert_eq!(args.get("enable_thinking"), Some(&serde_json::json!(true))); + } + + #[test] + fn test_responses_without_chat_template_args_converts_to_none() { + let req: NvCreateResponse = serde_json::from_value(serde_json::json!({ + "model": "dummy-model", + "input": "hello" + })) + .unwrap(); + + let chat_req = NvCreateChatCompletionRequest::try_from(req).unwrap(); + + assert!(chat_req.chat_template_args.is_none()); + } + + #[test] + fn test_responses_chat_template_args_rejects_non_object() { + let err = serde_json::from_value::(serde_json::json!({ + "model": "dummy-model", + "input": "hello", + "chat_template_args": "enable_thinking" + })); + + assert!(err.is_err()); + } + #[test] fn test_instructions_prepended_as_system_message() { let req = NvCreateResponse { @@ -1434,6 +1500,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1478,6 +1545,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1557,6 +1625,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1598,6 +1667,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1641,6 +1711,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1689,6 +1760,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1728,6 +1800,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1762,6 +1835,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1804,6 +1878,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1865,6 +1940,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1937,6 +2013,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -1994,6 +2071,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -2049,6 +2127,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -2096,6 +2175,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -2161,6 +2241,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -2240,6 +2321,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); @@ -2313,6 +2395,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); let messages = &chat_req.inner.messages; @@ -2382,6 +2465,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); let messages = &chat_req.inner.messages; @@ -2448,6 +2532,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); let messages = &chat_req.inner.messages; @@ -2505,6 +2590,7 @@ mod tests { ..Default::default() }, nvext: None, + chat_template_args: None, }; let chat_req: NvCreateChatCompletionRequest = req.try_into().unwrap(); let messages = &chat_req.inner.messages;