From 8b3dbafa5b93446b37aea5632c6ff07abf14a547 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Wed, 12 Aug 2026 19:40:27 +0530 Subject: [PATCH 1/4] Update Signed-off-by: Sai Kiran Polisetty --- lib/llm/src/http/service/openai.rs | 25 +++++- lib/llm/src/http/service/service_v2.rs | 118 +++++++++++++++++++++++++ 2 files changed, 141 insertions(+), 2 deletions(-) diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 10d1f9ed4ad0..580d44d5eddb 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -13,7 +13,7 @@ use axum::{ body::Body, extract::State, http::Request, - http::{HeaderMap, StatusCode}, + http::{HeaderMap, Method, StatusCode, Uri}, middleware::{self, Next}, response::{ IntoResponse, Response, @@ -2950,8 +2950,11 @@ pub fn validate_completion_fields_generic( async fn handler_responses( State((state, template)): State<(Arc, Option)>, headers: HeaderMap, - Json(mut request): Json, + body: Bytes, ) -> Result { + ensure_json_content_type(&headers)?; + let mut request: NvCreateResponse = parse_json_request("responses", &body)?; + // return a 503 if the service or model is not ready. // Resolve the templated model first so empty/missing `model` fields // don't bypass the gate. @@ -3405,6 +3408,24 @@ pub(crate) fn check_ready(state: &Arc) -> Result<(), ErrorRes Ok(()) } +/// Returns an OpenAI-compatible JSON error response for unmatched routes. +/// Axum's default fallback response has an empty body and does not set a +/// `Content-Type` header. +/// This fallback returns `404` with the standard `ErrorMessage` response format. +pub(crate) async fn unmatched_route_fallback(method: Method, uri: Uri) -> ErrorResponse { + let code = StatusCode::NOT_FOUND; + ( + code, + Json(ErrorMessage { + message: format!("Invalid URL ({} {})", method, uri.path()), + error_type: map_error_code_to_error_type(code), + code: code.as_u16(), + details: None, + metric_error_type: None, + }), + ) +} + /// Canonical, customer-facing message for "model is registered but not yet /// ready to serve requests" (deployment still initializing or incomplete). /// diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index 1f3ba362be2d..671f92833767 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -1250,6 +1250,10 @@ impl HttpServiceConfigBuilder { let router = system_router.merge(inference_router); + // Return an OpenAI-compatible JSON error for unmatched routes. + // Configure this after merging routers and before applying layers. + let router = router.fallback(super::openai::unmatched_route_fallback); + // Echo x-request-id from request to response headers for client correlation let router = router.layer(axum::middleware::from_fn(echo_request_id_header)); @@ -1639,6 +1643,120 @@ mod tests { handle.abort(); } + /// Starts a default `HttpService` on an ephemeral local port. + /// These tests exercise request handling before the readiness check, + /// so no model registration is required. + async fn spawn_default_service() -> (u16, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let service = HttpService::builder().port(port).build().unwrap(); + let handle = tokio::spawn(async move { + service + .run_with_listener(CancellationToken::new(), listener) + .await + .ok(); + }); + // Allow the server to begin accepting connections. + tokio::time::sleep(Duration::from_millis(20)).await; + (port, handle) + } + + /// Verifies that an unsupported content type returns the standard JSON error + /// envelope instead of Axum's default plain-text extractor rejection. + #[tokio::test] + async fn test_responses_non_json_content_type_returns_json_error() { + let (port, handle) = spawn_default_service().await; + + let resp = reqwest::Client::new() + .post(format!("http://localhost:{port}/v1/responses")) + .header("content-type", "text/plain") + .body(r#"{"model":"model","input":"hi"}"#) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), reqwest::StatusCode::UNSUPPORTED_MEDIA_TYPE); + let body: serde_json::Value = resp.json().await.expect("body must be JSON"); + assert_eq!(body["code"], 415); + assert_eq!( + body["message"], + "Expected request with Content-Type application/json" + ); + + handle.abort(); + } + + /// Verifies that malformed JSON returns the standard JSON error envelope + /// instead of Axum's default plain-text extractor rejection. + #[tokio::test] + async fn test_responses_malformed_json_returns_json_error() { + let (port, handle) = spawn_default_service().await; + + let resp = reqwest::Client::new() + .post(format!("http://localhost:{port}/v1/responses")) + .header("content-type", "application/json") + .body(r#"{"model":"model","input":"#) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); + let body: serde_json::Value = resp.json().await.expect("body must be JSON"); + assert_eq!(body["code"], 400); + assert!( + body["message"] + .as_str() + .expect("message must be a string") + .starts_with("Failed to deserialize the JSON body into the target type"), + "unexpected message: {}", + body["message"] + ); + + handle.abort(); + } + + /// Verifies that an unknown response ID returns a JSON `404 Not Found` + /// response instead of Axum's default empty fallback response. + #[tokio::test] + async fn test_unknown_response_id_returns_json_404() { + let (port, handle) = spawn_default_service().await; + + let resp = reqwest::Client::new() + .get(format!("http://localhost:{port}/v1/responses/resp_missing")) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), reqwest::StatusCode::NOT_FOUND); + let body: serde_json::Value = resp.json().await.expect("body must be JSON"); + assert_eq!(body["code"], 404); + assert_eq!( + body["message"], + "Invalid URL (GET /v1/responses/resp_missing)" + ); + + handle.abort(); + } + + /// Verifies that an unmatched route returns the standard JSON `404 Not Found` + /// error envelope. + #[tokio::test] + async fn test_unknown_path_returns_json_404() { + let (port, handle) = spawn_default_service().await; + + let resp = reqwest::Client::new() + .get(format!("http://localhost:{port}/v1/not_a_route")) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), reqwest::StatusCode::NOT_FOUND); + let body: serde_json::Value = resp.json().await.expect("body must be JSON"); + assert_eq!(body["code"], 404); + + handle.abort(); + } + fn make_chat_engine() -> crate::types::openai::chat_completions::OpenAIChatCompletionsStreamingEngine { Arc::new(crate::engines::StreamingEngineAdapter::new( From 678060865d786e5e958873efac7c3b7aee1bd506 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Wed, 12 Aug 2026 22:11:03 +0530 Subject: [PATCH 2/4] Update Signed-off-by: Sai Kiran Polisetty --- lib/llm/src/http/service/service_v2.rs | 32 ++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index 671f92833767..9cb9ced33836 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -1226,6 +1226,10 @@ impl HttpServiceConfigBuilder { append_route_docs(&mut all_docs, &mut seen_route_docs, route_docs)?; inference_router = inference_router.merge(route); } + + // Return an OpenAI-compatible JSON error for unmatched routes. + inference_router = inference_router.fallback(super::openai::unmatched_route_fallback); + inference_router = inference_router.layer( TraceLayer::new_for_http() .make_span_with(make_inference_request_span) @@ -1250,10 +1254,6 @@ impl HttpServiceConfigBuilder { let router = system_router.merge(inference_router); - // Return an OpenAI-compatible JSON error for unmatched routes. - // Configure this after merging routers and before applying layers. - let router = router.fallback(super::openai::unmatched_route_fallback); - // Echo x-request-id from request to response headers for client correlation let router = router.layer(axum::middleware::from_fn(echo_request_id_header)); @@ -1757,6 +1757,30 @@ mod tests { handle.abort(); } + /// Verifies that unsupported methods return `405 Method Not Allowed` with the + /// expected `Allow` header instead of the unmatched-route fallback `404`. + #[tokio::test] + async fn test_registered_route_with_unsupported_method_returns_405() { + let (port, handle) = spawn_default_service().await; + + let resp = reqwest::Client::new() + .get(format!("http://localhost:{port}/v1/responses")) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), reqwest::StatusCode::METHOD_NOT_ALLOWED); + assert_eq!( + resp.headers() + .get(reqwest::header::ALLOW) + .and_then(|value| value.to_str().ok()), + Some("POST"), + "expected Allow header to contain POST" + ); + + handle.abort(); + } + fn make_chat_engine() -> crate::types::openai::chat_completions::OpenAIChatCompletionsStreamingEngine { Arc::new(crate::engines::StreamingEngineAdapter::new( From 9f67a3b13358959aa92c3ba41eb3dcf43aa65198 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 18 Aug 2026 20:01:54 +0530 Subject: [PATCH 3/4] Update Signed-off-by: Sai Kiran Polisetty --- Cargo.lock | 1 + lib/llm/Cargo.toml | 1 + lib/llm/src/http/service/anthropic.rs | 17 +- lib/llm/src/http/service/openai.rs | 78 +++++++-- lib/llm/src/http/service/service_v2.rs | 223 ++++++++++++++++++++++--- 5 files changed, 284 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 394c55c1b00a..cc11168fa8cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2706,6 +2706,7 @@ dependencies = [ "futures-util", "galil-seiferas", "hf-hub 0.4.3", + "http-body-util", "image", "indicatif 0.18.4", "insta", diff --git a/lib/llm/Cargo.toml b/lib/llm/Cargo.toml index 7c774c071a84..970576a2e9cb 100644 --- a/lib/llm/Cargo.toml +++ b/lib/llm/Cargo.toml @@ -89,6 +89,7 @@ either = { workspace = true } futures = { workspace = true } futures-util = { workspace = true } hf-hub = { workspace = true, features = ["tokio", "rustls-tls", "ureq"] } +http-body-util = "0.1" ipnet = { workspace = true } libc = { workspace = true } lru = { workspace = true } diff --git a/lib/llm/src/http/service/anthropic.rs b/lib/llm/src/http/service/anthropic.rs index 41da1d588c38..69ee05f85050 100644 --- a/lib/llm/src/http/service/anthropic.rs +++ b/lib/llm/src/http/service/anthropic.rs @@ -16,7 +16,7 @@ use axum::{ Json, Router, body::Body, extract::State, - http::{HeaderMap, Request, StatusCode}, + http::{HeaderMap, Method, Request, StatusCode, Uri}, middleware::{self, Next}, response::{ IntoResponse, Response, @@ -68,13 +68,16 @@ use super::openai::{get_body_limit, get_or_create_request_id}; // Router // --------------------------------------------------------------------------- +/// Default route for the Anthropic Messages API when no override is configured. +pub(crate) const DEFAULT_MESSAGES_PATH: &str = "/v1/messages"; + /// Creates the router for the `/v1/messages` and `/v1/messages/count_tokens` endpoints. pub fn anthropic_messages_router( state: Arc, template: Option, path: Option, ) -> (Vec, Router) { - let path = path.unwrap_or("/v1/messages".to_string()); + let path = path.unwrap_or_else(|| DEFAULT_MESSAGES_PATH.to_string()); let count_tokens_path = format!("{}/count_tokens", &path); let doc = RouteDoc::new(axum::http::Method::POST, &path); let count_doc = RouteDoc::new(axum::http::Method::POST, &count_tokens_path); @@ -1132,6 +1135,16 @@ fn anthropic_error(status: StatusCode, error_type: &str, message: &str) -> Respo .into_response() } +/// Returns an Anthropic-compatible JSON `404` error response for an unmatched route. +/// Anthropic clients expect the nested `{"type": "error", "error": {...}}` +pub(crate) fn unmatched_route_response(method: &Method, uri: &Uri) -> Response { + anthropic_error( + StatusCode::NOT_FOUND, + "not_found_error", + &format!("Route not found: {} {}", method, uri.path()), + ) +} + #[cfg(test)] mod tests { use super::*; diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 8eb8db99538c..9060d0898bbd 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -29,6 +29,7 @@ use dynamo_runtime::{ protocols::annotated::AnnotationsProvider, }; use futures::{StreamExt, stream}; +use http_body_util::LengthLimitError; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use super::{ @@ -704,9 +705,9 @@ fn warn_nvext_disabled(endpoint: &str, nvext_present: bool, headers: &HeaderMap) async fn handler_completions( State(state): State>, headers: HeaderMap, - body: Bytes, + body: Body, ) -> Result { - ensure_json_content_type(&headers)?; + let body = read_json_request_body(&headers, body).await?; let mut request: NvCreateCompletionRequest = parse_json_request("completions", &body)?; if *FORCE_INCLUDE_USAGE && request.inner.stream.unwrap_or(false) { delta_common::force_include_usage(&mut request.inner.stream_options); @@ -1866,9 +1867,9 @@ async fn pooling( async fn handler_chat_completions( State((state, template)): State<(Arc, Option)>, headers: HeaderMap, - body: Bytes, + body: Body, ) -> Result { - ensure_json_content_type(&headers)?; + let body = read_json_request_body(&headers, body).await?; let mut request: NvCreateChatCompletionRequest = parse_json_request("chat completions", &body)?; if *FORCE_INCLUDE_USAGE && request.inner.stream.unwrap_or(false) { delta_common::force_include_usage(&mut request.inner.stream_options); @@ -2031,6 +2032,61 @@ fn unsupported_media_type_error() -> ErrorResponse { ) } +/// Returns the standard error response for a request body that exceeds the +/// configured size limit. +fn payload_too_large_error() -> ErrorResponse { + let code = StatusCode::PAYLOAD_TOO_LARGE; + ( + code, + Json(ErrorMessage { + message: format!( + "Request body exceeds the limit of {} MB set by {}", + get_body_limit() / (1024 * 1024), + env_llm::DYN_HTTP_BODY_LIMIT_MB + ), + error_type: map_error_code_to_error_type(code), + code: code.as_u16(), + details: None, + metric_error_type: None, + }), + ) +} + +/// Returns the standard error response when the request body cannot be read. +fn failed_to_read_request_body_error() -> ErrorResponse { + let code = StatusCode::BAD_REQUEST; + ( + code, + Json(ErrorMessage { + message: "Failed to read request body".to_string(), + error_type: map_error_code_to_error_type(code), + code: code.as_u16(), + details: None, + metric_error_type: None, + }), + ) +} + +/// Reads and buffers a JSON request body. +/// +/// Validates the `Content-Type` before reading the body and limits buffering to [`get_body_limit`]. +async fn read_json_request_body(headers: &HeaderMap, body: Body) -> Result { + ensure_json_content_type(headers)?; + axum::body::to_bytes(body, get_body_limit()) + .await + .map_err(|error| { + // `to_bytes` wraps an oversized-body failure in its error source + // rather than returning `LengthLimitError` directly. + if std::error::Error::source(&error) + .is_some_and(|source| source.is::()) + { + payload_too_large_error() + } else { + failed_to_read_request_body_error() + } + }) +} + fn is_json_content_type(content_type: &str) -> bool { let media_type = content_type.split(';').next().unwrap_or_default().trim(); let Some((media_type, subtype)) = media_type.split_once('/') else { @@ -3055,9 +3111,9 @@ pub fn validate_completion_fields_generic( async fn handler_responses( State((state, template)): State<(Arc, Option)>, headers: HeaderMap, - body: Bytes, + body: Body, ) -> Result { - ensure_json_content_type(&headers)?; + let body = read_json_request_body(&headers, body).await?; let mut request: NvCreateResponse = parse_json_request("responses", &body)?; // return a 503 if the service or model is not ready. @@ -3528,16 +3584,14 @@ pub(crate) fn check_ready(state: &Arc) -> Result<(), ErrorRes Ok(()) } -/// Returns an OpenAI-compatible JSON error response for unmatched routes. -/// Axum's default fallback response has an empty body and does not set a -/// `Content-Type` header. -/// This fallback returns `404` with the standard `ErrorMessage` response format. -pub(crate) async fn unmatched_route_fallback(method: Method, uri: Uri) -> ErrorResponse { +/// Returns an OpenAI-compatible JSON `404` error response for an +/// unmatched route. +pub(crate) fn unmatched_route_response(method: &Method, uri: &Uri) -> ErrorResponse { let code = StatusCode::NOT_FOUND; ( code, Json(ErrorMessage { - message: format!("Invalid URL ({} {})", method, uri.path()), + message: format!("Route not found: {} {}", method, uri.path()), error_type: map_error_code_to_error_type(code), code: code.as_u16(), details: None, diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index e10c8ef3be8d..dee46c0e9139 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -67,6 +67,39 @@ async fn echo_request_id_header( response } +/// State used to select the error format for unmatched routes. +#[derive(Clone)] +struct UnmatchedRouteState { + /// Base path of the Anthropic Messages API, or `None` when those endpoints + /// are disabled and every miss belongs to the OpenAI surface. + anthropic_path: Option>, +} + +/// Returns a protocol-compatible JSON `404` error response for an +/// unmatched route. +/// +/// Requests under the configured Anthropic Messages path receive an Anthropic +/// error envelope. All other requests receive an OpenAI-compatible envelope. +async fn unmatched_route_fallback( + axum::extract::State(state): axum::extract::State, + method: axum::http::Method, + uri: axum::http::Uri, +) -> axum::response::Response { + match state.anthropic_path.as_deref() { + Some(path) if path_within_namespace(uri.path(), path) => { + super::anthropic::unmatched_route_response(&method, &uri) + } + _ => super::openai::unmatched_route_response(&method, &uri).into_response(), + } +} + +/// Returns whether `path` is `namespace` or a route beneath it. +fn path_within_namespace(path: &str, namespace: &str) -> bool { + let namespace = namespace.trim_end_matches('/'); + path.strip_prefix(namespace) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('/')) +} + async fn track_inflight_inference( axum::extract::State(state): axum::extract::State>, request: axum::extract::Request, @@ -1255,10 +1288,6 @@ impl HttpServiceConfigBuilder { append_route_docs(&mut all_docs, &mut seen_route_docs, route_docs)?; inference_router = inference_router.merge(route); } - - // Return an OpenAI-compatible JSON error for unmatched routes. - inference_router = inference_router.fallback(super::openai::unmatched_route_fallback); - inference_router = inference_router.layer( TraceLayer::new_for_http() .make_span_with(make_inference_request_span) @@ -1283,6 +1312,27 @@ impl HttpServiceConfigBuilder { let router = system_router.merge(inference_router); + // Return protocol-compatible JSON errors for unmatched routes. Register this router + // outside `track_inflight_inference` so unmatched requests do not acquire an + // inference permit or return `503` while the service is draining. + let unmatched_router = axum::Router::new() + .fallback(unmatched_route_fallback) + .with_state(UnmatchedRouteState { + anthropic_path: anthropic_endpoints_enabled.then(|| { + var(HTTP_SVC_ANTHROPIC_PATH_ENV) + .unwrap_or_else(|_| super::anthropic::DEFAULT_MESSAGES_PATH.to_string()) + .into() + }), + }) + .layer( + // Use the inference span maker so 404s retain method, URI, and request ID + // at the default log level. + TraceLayer::new_for_http() + .make_span_with(make_inference_request_span) + .on_response(on_response), + ); + let router = router.fallback_service(unmatched_router); + // Echo x-request-id from request to response headers for client correlation let router = router.layer(axum::middleware::from_fn(echo_request_id_header)); @@ -1702,26 +1752,39 @@ mod tests { handle.abort(); } - /// Starts a default `HttpService` on an ephemeral local port. - /// These tests exercise request handling before the readiness check, - /// so no model registration is required. - async fn spawn_default_service() -> (u16, tokio::task::JoinHandle<()>) { + /// Starts an `HttpService` on an ephemeral local port. + /// + /// Applies `configure` before starting the service. These tests run before the + /// readiness check, so no model registration is required. + async fn spawn_service( + configure: impl FnOnce(HttpServiceConfigBuilder) -> HttpServiceConfigBuilder, + ) -> (u16, Arc, tokio::task::JoinHandle<()>) { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let port = listener.local_addr().unwrap().port(); - let service = HttpService::builder().port(port).build().unwrap(); + let service = configure(HttpService::builder().port(port)) + .build() + .unwrap(); + let state = service.state_clone(); let handle = tokio::spawn(async move { service .run_with_listener(CancellationToken::new(), listener) .await .ok(); }); + // Allow the server to begin accepting connections. tokio::time::sleep(Duration::from_millis(20)).await; + + (port, state, handle) + } + + async fn spawn_default_service() -> (u16, tokio::task::JoinHandle<()>) { + let (port, _, handle) = spawn_service(|builder| builder).await; (port, handle) } /// Verifies that an unsupported content type returns the standard JSON error - /// envelope instead of Axum's default plain-text extractor rejection. + /// envelope instead of Axum's default plain-text rejection. #[tokio::test] async fn test_responses_non_json_content_type_returns_json_error() { let (port, handle) = spawn_default_service().await; @@ -1746,7 +1809,7 @@ mod tests { } /// Verifies that malformed JSON returns the standard JSON error envelope - /// instead of Axum's default plain-text extractor rejection. + /// instead of Axum's default plain-text rejection. #[tokio::test] async fn test_responses_malformed_json_returns_json_error() { let (port, handle) = spawn_default_service().await; @@ -1774,8 +1837,7 @@ mod tests { handle.abort(); } - /// Verifies that an unknown response ID returns a JSON `404 Not Found` - /// response instead of Axum's default empty fallback response. + /// Verifies that an unknown response ID returns a JSON `404 Not Found` response. #[tokio::test] async fn test_unknown_response_id_returns_json_404() { let (port, handle) = spawn_default_service().await; @@ -1791,33 +1853,150 @@ mod tests { assert_eq!(body["code"], 404); assert_eq!( body["message"], - "Invalid URL (GET /v1/responses/resp_missing)" + "Route not found: GET /v1/responses/resp_missing" ); handle.abort(); } - /// Verifies that an unmatched route returns the standard JSON `404 Not Found` - /// error envelope. + /// Verifies that an oversized request body returns a JSON `413 Payload Too Large` + /// response. #[tokio::test] - async fn test_unknown_path_returns_json_404() { - let (port, handle) = spawn_default_service().await; + #[serial_test::serial] + async fn test_oversized_body_returns_json_413() { + temp_env::async_with_vars([(env_llm::DYN_HTTP_BODY_LIMIT_MB, Some("1"))], async move { + let (port, handle) = spawn_default_service().await; - let resp = reqwest::Client::new() + let oversized = serde_json::json!({ + "model": "model", + "input": "x".repeat(2 * 1024 * 1024), + }); + let resp = reqwest::Client::new() + .post(format!("http://localhost:{port}/v1/responses")) + .header("content-type", "application/json") + .body(serde_json::to_vec(&oversized).unwrap()) + .send() + .await + .expect("request failed"); + + assert_eq!(resp.status(), reqwest::StatusCode::PAYLOAD_TOO_LARGE); + let body: serde_json::Value = resp.json().await.expect("body must be JSON"); + assert_eq!(body["code"], 413); + assert_eq!( + body["message"], + format!( + "Request body exceeds the limit of 1 MB set by {}", + env_llm::DYN_HTTP_BODY_LIMIT_MB + ) + ); + + handle.abort(); + }) + .await; + } + + /// Verifies that unmatched routes return `404 Not Found` while registered + /// inference routes return `503 Service Unavailable` during draining. + #[tokio::test] + async fn test_unmatched_route_while_draining_returns_json_404() { + let (port, state, handle) = spawn_service(|builder| builder).await; + state.start_draining(); + + let client = reqwest::Client::new(); + let unmatched = client .get(format!("http://localhost:{port}/v1/not_a_route")) .send() .await .expect("request failed"); + assert_eq!(unmatched.status(), reqwest::StatusCode::NOT_FOUND); + let body: serde_json::Value = unmatched.json().await.expect("body must be JSON"); + assert_eq!(body["code"], 404); + + let registered = client + .post(format!("http://localhost:{port}/v1/responses")) + .json(&serde_json::json!({"model": "model", "input": "hi"})) + .send() + .await + .expect("request failed"); + + assert_eq!( + registered.status(), + reqwest::StatusCode::SERVICE_UNAVAILABLE, + "draining must still reject registered inference routes" + ); + + handle.abort(); + } + + /// Verifies that unmatched routes under the Anthropic Messages API path return + /// the Anthropic error envelope. + #[tokio::test] + async fn test_unmatched_anthropic_route_returns_anthropic_envelope() { + let (port, _state, handle) = + spawn_service(|builder| builder.enable_anthropic_endpoints(true)).await; + + let resp = reqwest::Client::new() + .get(format!("http://localhost:{port}/v1/messages/missing")) + .header("anthropic-version", "2023-06-01") + .send() + .await + .expect("request failed"); + assert_eq!(resp.status(), reqwest::StatusCode::NOT_FOUND); let body: serde_json::Value = resp.json().await.expect("body must be JSON"); - assert_eq!(body["code"], 404); + assert_eq!(body["type"], "error"); + assert_eq!(body["error"]["type"], "not_found_error"); + assert_eq!( + body["error"]["message"], + "Route not found: GET /v1/messages/missing" + ); + + let openai_response = reqwest::Client::new() + .get(format!("http://localhost:{port}/v1/messages_beta/missing")) + .send() + .await + .expect("request failed"); + + assert_eq!(openai_response.status(), reqwest::StatusCode::NOT_FOUND); + let openai_body: serde_json::Value = + openai_response.json().await.expect("body must be JSON"); + assert_eq!(openai_body["code"], 404); + assert_eq!( + openai_body["message"], + "Route not found: GET /v1/messages_beta/missing" + ); + assert!( + openai_body.get("error").is_none(), + "sibling paths must use the OpenAI error envelope" + ); handle.abort(); } - /// Verifies that unsupported methods return `405 Method Not Allowed` with the - /// expected `Allow` header instead of the unmatched-route fallback `404`. + /// Verifies that Anthropic fallback routing matches complete path segments. + #[test] + fn test_path_within_namespace() { + assert!(path_within_namespace("/v1/messages", "/v1/messages")); + assert!(path_within_namespace("/v1/messages/", "/v1/messages")); + assert!(path_within_namespace( + "/v1/messages/missing", + "/v1/messages" + )); + assert!(path_within_namespace( + "/v1/messages/missing", + "/v1/messages/" + )); + assert!(!path_within_namespace("/v1/messages_beta", "/v1/messages")); + assert!(!path_within_namespace("/v1/messages-v2", "/v1/messages")); + assert!(!path_within_namespace( + "/v1/chat/completions", + "/v1/messages" + )); + } + + /// Verifies that `GET /v1/responses` returns `405 Method Not Allowed` and an + /// `Allow: POST` header. #[tokio::test] async fn test_registered_route_with_unsupported_method_returns_405() { let (port, handle) = spawn_default_service().await; From 8f65456989bba9c1e8d7c89dbf22d021ddaf90b9 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 18 Aug 2026 21:17:01 +0530 Subject: [PATCH 4/4] Update Cargo.lock Signed-off-by: Sai Kiran Polisetty --- lib/bindings/kvbm/Cargo.lock | 1 + lib/bindings/python/Cargo.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index f442b24ddad7..806055ed3090 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -1750,6 +1750,7 @@ dependencies = [ "futures-util", "galil-seiferas", "hf-hub", + "http-body-util", "image", "ipnet", "json-five", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index 1c7a8a143e82..6ece8acbeae9 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -1813,6 +1813,7 @@ dependencies = [ "futures-util", "galil-seiferas", "hf-hub 0.4.3", + "http-body-util", "image", "ipnet", "json-five",