Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions lib/bindings/kvbm/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions lib/bindings/python/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions lib/llm/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ either = { workspace = true }
futures = { workspace = true }
futures-util = { workspace = true }
hf-hub = { workspace = true, features = ["tokio", "rustls-tls", "ureq"] }
http-body-util = "0.1"
ipnet = { workspace = true }
libc = { workspace = true }
lru = { workspace = true }
Expand Down
17 changes: 15 additions & 2 deletions lib/llm/src/http/service/anthropic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use axum::{
Json, Router,
body::Body,
extract::State,
http::{HeaderMap, Request, StatusCode},
http::{HeaderMap, Method, Request, StatusCode, Uri},
middleware::{self, Next},
response::{
IntoResponse, Response,
Expand Down Expand Up @@ -68,13 +68,16 @@ use super::openai::{get_body_limit, get_or_create_request_id};
// Router
// ---------------------------------------------------------------------------

/// Default route for the Anthropic Messages API when no override is configured.
pub(crate) const DEFAULT_MESSAGES_PATH: &str = "/v1/messages";

/// Creates the router for the `/v1/messages` and `/v1/messages/count_tokens` endpoints.
pub fn anthropic_messages_router(
state: Arc<service_v2::State>,
template: Option<RequestTemplate>,
path: Option<String>,
) -> (Vec<RouteDoc>, Router) {
let path = path.unwrap_or("/v1/messages".to_string());
let path = path.unwrap_or_else(|| DEFAULT_MESSAGES_PATH.to_string());
let count_tokens_path = format!("{}/count_tokens", &path);
let doc = RouteDoc::new(axum::http::Method::POST, &path);
let count_doc = RouteDoc::new(axum::http::Method::POST, &count_tokens_path);
Expand Down Expand Up @@ -1132,6 +1135,16 @@ fn anthropic_error(status: StatusCode, error_type: &str, message: &str) -> Respo
.into_response()
}

/// Returns an Anthropic-compatible JSON `404` error response for an unmatched route.
/// Anthropic clients expect the nested `{"type": "error", "error": {...}}`
pub(crate) fn unmatched_route_response(method: &Method, uri: &Uri) -> Response {
anthropic_error(
StatusCode::NOT_FOUND,
"not_found_error",
&format!("Route not found: {} {}", method, uri.path()),
)
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
87 changes: 81 additions & 6 deletions lib/llm/src/http/service/openai.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use axum::{
body::Body,
extract::State,
http::Request,
http::{HeaderMap, StatusCode},
http::{HeaderMap, Method, StatusCode, Uri},
middleware::{self, Next},
response::{
IntoResponse, Response,
Expand All @@ -29,6 +29,7 @@ use dynamo_runtime::{
protocols::annotated::AnnotationsProvider,
};
use futures::{StreamExt, stream};
use http_body_util::LengthLimitError;
use serde::{Deserialize, Serialize, de::DeserializeOwned};

use super::{
Expand Down Expand Up @@ -704,9 +705,9 @@ fn warn_nvext_disabled(endpoint: &str, nvext_present: bool, headers: &HeaderMap)
async fn handler_completions(
State(state): State<Arc<service_v2::State>>,
headers: HeaderMap,
body: Bytes,
body: Body,
) -> Result<Response, ErrorResponse> {
ensure_json_content_type(&headers)?;
let body = read_json_request_body(&headers, body).await?;
let mut request: NvCreateCompletionRequest = parse_json_request("completions", &body)?;
if *FORCE_INCLUDE_USAGE && request.inner.stream.unwrap_or(false) {
delta_common::force_include_usage(&mut request.inner.stream_options);
Expand Down Expand Up @@ -1866,9 +1867,9 @@ async fn pooling(
async fn handler_chat_completions(
State((state, template)): State<(Arc<service_v2::State>, Option<RequestTemplate>)>,
headers: HeaderMap,
body: Bytes,
body: Body,
) -> Result<Response, ErrorResponse> {
ensure_json_content_type(&headers)?;
let body = read_json_request_body(&headers, body).await?;
let mut request: NvCreateChatCompletionRequest = parse_json_request("chat completions", &body)?;
if *FORCE_INCLUDE_USAGE && request.inner.stream.unwrap_or(false) {
delta_common::force_include_usage(&mut request.inner.stream_options);
Expand Down Expand Up @@ -2031,6 +2032,61 @@ fn unsupported_media_type_error() -> ErrorResponse {
)
}

/// Returns the standard error response for a request body that exceeds the
/// configured size limit.
fn payload_too_large_error() -> ErrorResponse {
let code = StatusCode::PAYLOAD_TOO_LARGE;
(
code,
Json(ErrorMessage {
message: format!(
"Request body exceeds the limit of {} MB set by {}",
get_body_limit() / (1024 * 1024),
env_llm::DYN_HTTP_BODY_LIMIT_MB
),
error_type: map_error_code_to_error_type(code),
code: code.as_u16(),
details: None,
metric_error_type: None,
}),
)
}

/// Returns the standard error response when the request body cannot be read.
fn failed_to_read_request_body_error() -> ErrorResponse {
let code = StatusCode::BAD_REQUEST;
(
code,
Json(ErrorMessage {
message: "Failed to read request body".to_string(),
error_type: map_error_code_to_error_type(code),
code: code.as_u16(),
details: None,
metric_error_type: None,
}),
)
}

/// Reads and buffers a JSON request body.
///
/// Validates the `Content-Type` before reading the body and limits buffering to [`get_body_limit`].
async fn read_json_request_body(headers: &HeaderMap, body: Body) -> Result<Bytes, ErrorResponse> {
ensure_json_content_type(headers)?;
axum::body::to_bytes(body, get_body_limit())
.await
.map_err(|error| {
// `to_bytes` wraps an oversized-body failure in its error source
// rather than returning `LengthLimitError` directly.
if std::error::Error::source(&error)
.is_some_and(|source| source.is::<LengthLimitError>())
{
payload_too_large_error()
} else {
failed_to_read_request_body_error()
}
})
}

fn is_json_content_type(content_type: &str) -> bool {
let media_type = content_type.split(';').next().unwrap_or_default().trim();
let Some((media_type, subtype)) = media_type.split_once('/') else {
Expand Down Expand Up @@ -3055,8 +3111,11 @@ pub fn validate_completion_fields_generic(
async fn handler_responses(
State((state, template)): State<(Arc<service_v2::State>, Option<RequestTemplate>)>,
headers: HeaderMap,
Json(mut request): Json<NvCreateResponse>,
body: Body,
) -> Result<Response, ErrorResponse> {
let body = read_json_request_body(&headers, body).await?;
let mut request: NvCreateResponse = parse_json_request("responses", &body)?;

// return a 503 if the service or model is not ready.
// Resolve the templated model first so empty/missing `model` fields
// don't bypass the gate.
Expand Down Expand Up @@ -3532,6 +3591,22 @@ pub(crate) fn check_ready(state: &Arc<service_v2::State>) -> Result<(), ErrorRes
Ok(())
}

/// Returns an OpenAI-compatible JSON `404` error response for an
/// unmatched route.
pub(crate) fn unmatched_route_response(method: &Method, uri: &Uri) -> ErrorResponse {
let code = StatusCode::NOT_FOUND;
(
code,
Json(ErrorMessage {
message: format!("Route not found: {} {}", method, uri.path()),
error_type: map_error_code_to_error_type(code),
code: code.as_u16(),
details: None,
metric_error_type: None,
}),
)
}

/// Canonical, customer-facing message for "model is registered but not yet
/// ready to serve requests" (deployment still initializing or incomplete).
///
Expand Down
Loading
Loading