From 2e75381ad3d8e00f18310172fa985e645af4428a Mon Sep 17 00:00:00 2001 From: Ryan McCormick Date: Thu, 30 Jul 2026 12:35:55 -0700 Subject: [PATCH 1/2] fix(frontend): return 400 for prompt template errors Signed-off-by: Ryan McCormick Signed-off-by: Krishnan Prashanth --- lib/llm/src/preprocessor.rs | 116 ++++++++++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index ebb96e07993e..2f9102e7f221 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -1461,17 +1461,17 @@ impl OpenAIPreprocessor { hidden_eos_token_ids.len() != before } + /// Rendering is driven by the request, so its failures are reported as 400 rather than + /// 500, matching vLLM. A misconfigured template can also fail here, for instance a + /// `chat_template` map that omits the `tool_use` key, so log the cause chain before it + /// is flattened into the client-facing message. fn map_prompt_render_error(error: anyhow::Error) -> anyhow::Error { - if let Some(PromptRenderError::InvalidRequest(message)) = - error.downcast_ref::() - { - return DynamoError::builder() - .error_type(ErrorType::InvalidArgument) - .message(message.clone()) - .build() - .into(); - } - error + tracing::debug!(?error, "Chat template rendering failed"); + let message = match error.downcast_ref::() { + Some(PromptRenderError::InvalidRequest(message)) => message.clone(), + None => format!("{error:#}"), + }; + invalid_argument_error(message) } pub fn apply_template< @@ -4233,13 +4233,16 @@ mod tests { } #[test] - fn ordinary_prompt_error_remains_internal() { + fn ordinary_prompt_error_maps_to_invalid_argument() { let mapped = OpenAIPreprocessor::map_prompt_render_error(anyhow::anyhow!( "template configuration failed" )); + let mapped = mapped + .downcast_ref::() + .expect("any prompt render failure should map to a DynamoError"); - assert!(mapped.downcast_ref::().is_none()); - assert_eq!(mapped.to_string(), "template configuration failed"); + assert!(matches!(mapped.error_type(), ErrorType::InvalidArgument)); + assert_eq!(mapped.message(), "template configuration failed"); } fn url_entry(u: &str) -> MultimodalData { @@ -5334,6 +5337,93 @@ mod tests { ); } + fn test_prompt_formatter(template: &str) -> Arc { + let template: dynamo_renderer::ChatTemplate = serde_json::from_value(serde_json::json!({ + "chat_template": template + })) + .unwrap(); + match dynamo_renderer::PromptFormatter::from_parts( + template, + dynamo_renderer::ContextMixins::default(), + false, + ) + .unwrap() + { + dynamo_renderer::PromptFormatter::OAI(formatter) => formatter, + } + } + + fn assistant_only_request() -> NvCreateChatCompletionRequest { + serde_json::from_value(serde_json::json!({ + "model": "test-model", + "messages": [{"role": "assistant", "content": "prefill"}] + })) + .unwrap() + } + + const REQUIRES_USER_TEMPLATE: &str = "\ + {% set ns = namespace(has_user=false) %}\ + {% for message in messages %}\ + {% if message['role'] == 'user' %}{% set ns.has_user = true %}{% endif %}\ + {% endfor %}\ + {% if not ns.has_user %}{{ raise_exception('No user query found in messages.') }}{% endif %}\ + {{ messages[0]['content'] }}"; + + fn render_through_preprocessor( + formatter: &dyn OAIPromptFormatter, + request: &dyn OAIChatLikeRequest, + ) -> Result { + formatter + .render_prompt(request) + .map_err(OpenAIPreprocessor::map_prompt_render_error) + } + + #[test] + fn test_assistant_only_request_accepted_when_template_accepts_it() { + let formatter = test_prompt_formatter( + "{% for message in messages %}{{ message['role'] }}:{{ message['content'] }}{% endfor %}", + ); + + let rendered = + render_through_preprocessor(formatter.as_ref(), &assistant_only_request()).unwrap(); + + assert_eq!(rendered.as_str(), "assistant:prefill"); + } + + #[test] + fn test_assistant_only_template_error_is_invalid_argument() { + let formatter = test_prompt_formatter(REQUIRES_USER_TEMPLATE); + + let error = render_through_preprocessor(formatter.as_ref(), &assistant_only_request()) + .context("Failed to apply prompt template") + .unwrap_err(); + let dynamo_error = error + .chain() + .find_map(|cause| cause.downcast_ref::()) + .expect("template render error should be classified as a DynamoError"); + + assert_eq!(dynamo_error.error_type(), ErrorType::InvalidArgument); + assert!( + dynamo_error + .message() + .contains("No user query found in messages.") + ); + } + + #[test] + fn test_restrictive_template_accepts_request_with_user_message() { + let formatter = test_prompt_formatter(REQUIRES_USER_TEMPLATE); + let request: NvCreateChatCompletionRequest = serde_json::from_value(serde_json::json!({ + "model": "test-model", + "messages": [{"role": "user", "content": "hello"}] + })) + .unwrap(); + + let rendered = render_through_preprocessor(formatter.as_ref(), &request).unwrap(); + + assert_eq!(rendered.as_str(), "hello"); + } + #[test] fn test_kimi_thinking_normalization_keeps_template_and_gates_in_sync() { let template: dynamo_renderer::ChatTemplate = serde_json::from_value(serde_json::json!({ From b642340f47341cc40cb7c0b84ba8d1e87d4ecc8e Mon Sep 17 00:00:00 2001 From: Krishnan Prashanth Date: Thu, 30 Jul 2026 12:51:36 -0700 Subject: [PATCH 2/2] test(llm): cover chat template render failure over HTTP Signed-off-by: Krishnan Prashanth --- lib/llm/tests/chat_template_render_errors.rs | 279 +++++++++++++++++++ 1 file changed, 279 insertions(+) create mode 100644 lib/llm/tests/chat_template_render_errors.rs diff --git a/lib/llm/tests/chat_template_render_errors.rs b/lib/llm/tests/chat_template_render_errors.rs new file mode 100644 index 000000000000..6ef109f56fe3 --- /dev/null +++ b/lib/llm/tests/chat_template_render_errors.rs @@ -0,0 +1,279 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! End-to-end coverage for chat template rendering failures over HTTP. +//! +//! Rendering only consumes the request, so a template that refuses to render is a client +//! error: the frontend must answer 400 with a JSON body on both the streaming and the +//! non-streaming path, and must not reject inputs the model's own template accepts. + +use std::sync::Arc; + +use anyhow::Error; +use dynamo_llm::http::service::service_v2::HttpService; +use dynamo_llm::model_card::ModelDeploymentCard; +use dynamo_llm::preprocessor::{BackendOutput, OpenAIPreprocessor, PreprocessedRequest}; +use dynamo_llm::protocols::Annotated; +use dynamo_llm::protocols::openai::chat_completions::{ + NvCreateChatCompletionRequest, NvCreateChatCompletionStreamResponse, +}; +use dynamo_runtime::CancellationToken; +use dynamo_runtime::pipeline::{ + AsyncEngine, AsyncEngineContextProvider, ManyOut, Operator, ResponseStream, SingleIn, + async_trait, +}; +use reqwest::StatusCode; + +#[path = "common/ports.rs"] +mod ports; + +use ports::bind_random_port; + +/// mock-llama carries a chat template in tokenizer_config.json, which the preprocessor +/// needs, and that template renders any message list without inspecting roles. +const MODEL_PATH: &str = "tests/data/sample-models/mock-llama-3.1-8b-instruct"; + +const ACCEPTING_MODEL: &str = "accepting-template"; +const REJECTING_MODEL: &str = "rejecting-template"; + +/// Mirrors the guard published chat templates use to refuse conversations they cannot +/// encode. `raise_exception` is minijinja's abort hook, so rendering fails mid-template. +const REQUIRES_USER_TEMPLATE: &str = "\ + {% set ns = namespace(has_user=false) %}\ + {% for message in messages %}\ + {% if message['role'] == 'user' %}{% set ns.has_user = true %}{% endif %}\ + {% endfor %}\ + {% if not ns.has_user %}{{ raise_exception('No user query found in messages.') }}{% endif %}\ + {{ messages[0]['content'] }}"; + +/// Terminal engine for the pipeline. The rejecting-template requests never reach it. +struct EchoBackend; + +#[async_trait] +impl AsyncEngine, ManyOut>, Error> + for EchoBackend +{ + async fn generate( + &self, + request: SingleIn, + ) -> Result>, Error> { + let (_request, context) = request.transfer(()); + let ctx = context.context(); + + let output = BackendOutput { + token_ids: vec![], + tokens: vec![], + text: Some("ok".to_string()), + cum_log_probs: None, + log_probs: None, + top_logprobs: None, + finish_reason: Some(dynamo_llm::protocols::common::FinishReason::Stop), + stop_reason: None, + index: Some(0), + completion_usage: None, + disaggregated_params: None, + encoder_result: None, + worker_trace_link: None, + engine_data: None, + routing_data: None, + }; + + Ok(ResponseStream::new( + Box::pin(futures::stream::once(async move { + Annotated::from_data(output) + })), + ctx, + )) + } +} + +/// Wires the preprocessor ahead of a backend the way the frontend does in production, so +/// chat template rendering happens inside the engine the HTTP service calls. +struct PreprocessingChatEngine { + preprocessor: Arc, + backend: Arc< + dyn AsyncEngine, ManyOut>, Error>, + >, +} + +impl PreprocessingChatEngine { + fn new(mdc: ModelDeploymentCard) -> Self { + Self { + preprocessor: OpenAIPreprocessor::new(mdc).expect("failed to build preprocessor"), + backend: Arc::new(EchoBackend), + } + } +} + +#[async_trait] +impl + AsyncEngine< + SingleIn, + ManyOut>, + Error, + > for PreprocessingChatEngine +{ + async fn generate( + &self, + request: SingleIn, + ) -> Result>, Error> { + Operator::generate(self.preprocessor.as_ref(), request, self.backend.clone()).await + } +} + +struct TestService { + port: u16, + client: reqwest::Client, + cancel: CancellationToken, + join: tokio::task::JoinHandle>, + // Holds the custom template on disk for the lifetime of the service. + _template: tempfile::NamedTempFile, +} + +impl TestService { + /// Registers the same model twice, differing only in chat template: one that renders + /// any history, and one that refuses a history without a user turn. + async fn start() -> Self { + let mut template = tempfile::Builder::new() + .suffix(".jinja") + .tempfile() + .expect("failed to create custom template file"); + std::io::Write::write_all(&mut template, REQUIRES_USER_TEMPLATE.as_bytes()) + .expect("failed to write custom template file"); + + let (listener, port) = bind_random_port().await; + let service = HttpService::builder() + .port(port) + .host("127.0.0.1") + .enable_chat_endpoints(true) + .build() + .expect("failed to build HTTP service"); + + for (model, custom_template) in [ + (ACCEPTING_MODEL, None), + (REJECTING_MODEL, Some(template.path())), + ] { + let mut mdc = ModelDeploymentCard::load_from_disk(MODEL_PATH, custom_template) + .expect("failed to load model deployment card"); + mdc.set_name(model); + service + .model_manager() + .add_chat_completions_model( + model, + mdc.mdcsum(), + Arc::new(PreprocessingChatEngine::new(mdc.clone())), + ) + .expect("failed to register model"); + } + + let cancel = CancellationToken::new(); + let join = service.spawn_with_listener(cancel.clone(), listener).await; + let client = reqwest::Client::builder() + .no_proxy() + .build() + .expect("failed to build HTTP client"); + + let service = Self { + port, + client, + cancel, + join, + _template: template, + }; + service.wait_for_health().await; + service + } + + async fn wait_for_health(&self) { + let url = format!("http://127.0.0.1:{}/health", self.port); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + if self + .client + .get(&url) + .send() + .await + .is_ok_and(|response| response.status().is_success()) + { + return; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("HTTP service did not become healthy"); + } + + async fn post_assistant_only(&self, model: &str, stream: bool) -> reqwest::Response { + self.client + .post(format!( + "http://127.0.0.1:{}/v1/chat/completions", + self.port + )) + .json(&serde_json::json!({ + "model": model, + "stream": stream, + "max_tokens": 1, + "messages": [{"role": "assistant", "content": "prefill"}] + })) + .send() + .await + .expect("POST /v1/chat/completions failed") + } + + async fn shutdown(self) { + self.cancel.cancel(); + self.join + .await + .expect("HTTP service task panicked") + .expect("HTTP service returned an error"); + } +} + +#[tokio::test] +async fn template_render_failure_returns_400_json_not_sse() { + let service = TestService::start().await; + + // Streaming matters here: the frontend commits to HTTP 200 before the first SSE frame, + // so a render failure has to surface as a status code rather than an error event. + for stream in [false, true] { + let response = service.post_assistant_only(REJECTING_MODEL, stream).await; + + assert_eq!( + response.status(), + StatusCode::BAD_REQUEST, + "stream={stream}" + ); + assert_eq!( + response.headers().get(reqwest::header::CONTENT_TYPE), + Some(&reqwest::header::HeaderValue::from_static( + "application/json" + )), + "stream={stream}" + ); + + let body: serde_json::Value = response.json().await.expect("error body was not JSON"); + assert_eq!(body["code"], 400, "stream={stream}"); + assert!( + body["message"] + .as_str() + .is_some_and(|message| message.contains("No user query found in messages.")), + "stream={stream}, body={body}" + ); + } + + service.shutdown().await; +} + +#[tokio::test] +async fn assistant_only_history_succeeds_when_template_accepts_it() { + let service = TestService::start().await; + + let response = service.post_assistant_only(ACCEPTING_MODEL, false).await; + + assert_eq!(response.status(), StatusCode::OK); + let body: serde_json::Value = response.json().await.expect("response body was not JSON"); + assert_eq!(body["choices"][0]["message"]["content"], "ok", "{body}"); + + service.shutdown().await; +}