Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/llm/src/http/service/anthropic.rs
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ async fn handler_anthropic_messages(
}

// Create request context
let request_id = get_or_create_request_id(None, &headers);
let request_id = get_or_create_request_id(&headers);
let streaming = request.stream;
let cancellation_labels = CancellationLabels {
model: request.model.clone(),
Expand Down
72 changes: 39 additions & 33 deletions lib/llm/src/http/service/openai.rs
Original file line number Diff line number Diff line change
Expand Up @@ -291,37 +291,43 @@ pub async fn smart_json_error_middleware(request: Request<Body>, next: Next) ->
}
}

/// Get the request ID from a primary source, or next from the headers, or lastly create a new one if not present
// TODO: Similar function exists in lib/llm/src/grpc/service/openai.rs but with different signature and simpler logic
pub(super) fn get_or_create_request_id(primary: Option<&str>, headers: &HeaderMap) -> String {
// Try to get request id from trace context
/// Validate the `x-dynamo-request-id` header and return the request ID.
///
/// If the header is present but invalid (not UTF-8 or not a UUID), a warning is logged
/// and a new UUID is generated instead of returning an error.
///
/// The request ID comes from the trace context — `make_inference_request_span()` guarantees it by
/// generating a UUID when the client doesn't provide one.
pub(super) fn get_or_create_request_id(headers: &HeaderMap) -> String {
// Validate x-dynamo-request-id header if present, warn on invalid values
if let Some(raw) = headers.get(DYNAMO_REQUEST_ID_HEADER) {
match raw.to_str() {
Err(_) => {
tracing::warn!(
"{} header must be a valid UTF-8 string",
DYNAMO_REQUEST_ID_HEADER
);
}
Ok(s) if uuid::Uuid::parse_str(s).is_err() => {
tracing::warn!(
"{} header must be a valid UUID, got: {}",
DYNAMO_REQUEST_ID_HEADER,
s
);
}
Ok(_) => {} // Valid header, will be picked up from trace context below
}
}

// Prefer trace context (set by make_inference_request_span via DistributedTraceIdLayer)
if let Some(trace_context) = get_distributed_tracing_context()
&& let Some(x_dynamo_request_id) = trace_context.x_dynamo_request_id
{
return x_dynamo_request_id;
}

// Try to get the request ID from the primary source
if let Some(primary) = primary
&& let Ok(uuid) = uuid::Uuid::parse_str(primary)
{
return uuid.to_string();
}

// Try to get the request ID header as a string slice
let request_id_opt = headers
.get(DYNAMO_REQUEST_ID_HEADER)
.and_then(|h| h.to_str().ok());

// Try to parse the request ID as a UUID, or generate a new one if missing/invalid
let uuid = match request_id_opt {
Some(request_id) => {
uuid::Uuid::parse_str(request_id).unwrap_or_else(|_| uuid::Uuid::new_v4())
}
None => uuid::Uuid::new_v4(),
};

uuid.to_string()
// Fallback: generate new UUID
uuid::Uuid::new_v4().to_string()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

/// OpenAI Completions Request Handler
Expand All @@ -343,7 +349,7 @@ async fn handler_completions(
request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers);

// create the context for the request
let request_id = get_or_create_request_id(request.inner.user.as_deref(), &headers);
let request_id = get_or_create_request_id(&headers);
let streaming = request.inner.stream.unwrap_or(false);
let cancellation_labels = CancellationLabels {
model: request.inner.model.clone(),
Expand Down Expand Up @@ -723,7 +729,7 @@ async fn embeddings(
// return a 503 if the service is not ready
check_ready(&state)?;

let request_id = get_or_create_request_id(request.inner.user.as_deref(), &headers);
let request_id = get_or_create_request_id(&headers);
let request = Context::with_id(request, request_id);
let request_id = request.id().to_string();

Expand Down Expand Up @@ -801,7 +807,7 @@ async fn handler_chat_completions(
request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers);

// create the context for the request
let request_id = get_or_create_request_id(request.inner.user.as_deref(), &headers);
let request_id = get_or_create_request_id(&headers);
let streaming = request.inner.stream.unwrap_or(false);
let cancellation_labels = CancellationLabels {
model: request.inner.model.clone(),
Expand Down Expand Up @@ -1410,7 +1416,7 @@ async fn handler_responses(
request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers);

// create the context for the request
let request_id = get_or_create_request_id(None, &headers);
let request_id = get_or_create_request_id(&headers);
let streaming = request.inner.stream.unwrap_or(false);
let cancellation_labels = CancellationLabels {
model: request.inner.model.clone().unwrap_or_default(),
Expand Down Expand Up @@ -1902,7 +1908,7 @@ async fn images(
// return a 503 if the service is not ready
check_ready(&state)?;

let request_id = get_or_create_request_id(request.inner.user.as_deref(), &headers);
let request_id = get_or_create_request_id(&headers);
let request = Context::with_id(request, request_id);
let request_id = request.id().to_string();

Expand Down Expand Up @@ -1995,7 +2001,7 @@ async fn videos(
// return a 503 if the service is not ready
check_ready(&state)?;

let request_id = get_or_create_request_id(request.user.as_deref(), &headers);
let request_id = get_or_create_request_id(&headers);
let request = Context::with_id(request, request_id);
let request_id = request.id().to_string();

Expand Down Expand Up @@ -2066,7 +2072,7 @@ async fn video_stream(
) -> Result<Response, ErrorResponse> {
check_ready(&state)?;

let request_id = get_or_create_request_id(request.user.as_deref(), &headers);
let request_id = get_or_create_request_id(&headers);
let request = Context::with_id(request, request_id);
let model = request.model.clone();

Expand Down Expand Up @@ -2220,7 +2226,7 @@ async fn audio_speech(
check_ready(&state)?;

let response_format = request.response_format.clone();
let request_id = get_or_create_request_id(request.user.as_deref(), &headers);
let request_id = get_or_create_request_id(&headers);
let request = Context::with_id(request, request_id);
let request_id = request.id().to_string();

Expand Down
110 changes: 68 additions & 42 deletions lib/llm/src/http/service/service_v2.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ use derive_builder::Builder;
use dynamo_runtime::config::env_is_truthy;
use dynamo_runtime::config::environment_names::llm as env_llm;
use dynamo_runtime::discovery::Discovery;
use dynamo_runtime::logging::make_request_span;
use dynamo_runtime::logging::{make_inference_request_span, make_system_request_span};
use dynamo_runtime::metrics::{
frontend_perf::ensure_frontend_perf_metrics_registered_prometheus,
request_plane::ensure_request_plane_metrics_registered_prometheus,
Expand All @@ -40,6 +40,19 @@ use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken;
use tower_http::trace::TraceLayer;

/// Middleware that echoes `x-request-id` from request to response headers.
async fn echo_request_id_header(
request: axum::extract::Request,
next: axum::middleware::Next,
) -> axum::response::Response {
let x_request_id = request.headers().get("x-request-id").cloned();
let mut response = next.run(request).await;
if let Some(value) = x_request_id {
response.headers_mut().insert("x-request-id", value);
}
response
}

/// HTTP service shared state
pub struct State {
metrics: Arc<Metrics>,
Expand Down Expand Up @@ -491,11 +504,37 @@ impl HttpServiceConfigBuilder {
tracing::warn!("Failed to register transport metrics: {}", e);
}

let mut router = axum::Router::new();

let mut all_docs = Vec::new();

let mut routes = vec![
// on_response callback for system routes (health, metrics, models)
let on_response_system = |response: &Response<Body>,
latency: Duration,
_span: &tracing::Span| {
let status = response.status();
let latency_ms = latency.as_millis();
if status.is_server_error() {
tracing::error!(status = %status.as_u16(), latency_ms = %latency_ms, "http response sent");
} else if status.is_client_error() {
tracing::warn!(status = %status.as_u16(), latency_ms = %latency_ms, "http response sent");
} else {
tracing::debug!(status = %status.as_u16(), latency_ms = %latency_ms, "http response sent");
}
};
// on_response callback for inference routes (completions, chat, embeddings, etc.)
let on_response_inference = |response: &Response<Body>,
latency: Duration,
_span: &tracing::Span| {
let status = response.status();
let latency_ms = latency.as_millis();
if status.is_server_error() || status.is_client_error() {
tracing::error!(status = %status.as_u16(), latency_ms = %latency_ms, "http response sent");
} else {
tracing::info!(status = %status.as_u16(), latency_ms = %latency_ms, "http response sent");
}
};

// System routes (health, metrics, models) — debug-level spans
let system_routes = vec![
metrics::router(
registry,
var(HTTP_SVC_METRICS_PATH_ENV).ok(),
Expand All @@ -506,54 +545,41 @@ impl HttpServiceConfigBuilder {
super::health::live_check_router(state.clone(), var(HTTP_SVC_LIVE_PATH_ENV).ok()),
super::busy_threshold::busy_threshold_router(state.clone(), None),
];
let mut system_router = axum::Router::new();
for (route_docs, route) in system_routes {
system_router = system_router.merge(route);
all_docs.extend(route_docs);
}
system_router = system_router.layer(
TraceLayer::new_for_http()
.make_span_with(make_system_request_span)
.on_response(on_response_system),
);

// Inference routes (completions, chat, embeddings, etc.) — info-level spans
let endpoint_routes =
HttpServiceConfigBuilder::get_endpoints_router(state.clone(), &config.request_template);
routes.extend(endpoint_routes);
for (route_docs, route) in routes {
router = router.merge(route);
let mut inference_router = axum::Router::new();
for (route_docs, route) in endpoint_routes {
inference_router = inference_router.merge(route);
all_docs.extend(route_docs);
}
inference_router = inference_router.layer(
TraceLayer::new_for_http()
.make_span_with(make_inference_request_span)
.on_response(on_response_inference),
);

// Add OpenAPI documentation routes (must be after all other routes so it can document them)
// Note: The path parameter is currently unused as SwaggerUi requires static paths
// OpenAPI documentation routes (system)
let (openapi_docs, openapi_route) =
super::openapi_docs::openapi_router(all_docs.clone(), None);
router = router.merge(openapi_route);
system_router = system_router.merge(openapi_route);
all_docs.extend(openapi_docs);

// Add span for tracing
// Add on_response callback for logging response status code
router = router.layer(
TraceLayer::new_for_http()
.make_span_with(make_request_span)
.on_response(
|response: &Response<Body>, latency: Duration, _span: &tracing::Span| {
let status = response.status();
let latency_ms = latency.as_millis();

if status.is_server_error() {
tracing::error!(
status = %status.as_u16(),
latency_ms = %latency_ms,
"request completed with server error"
);
} else if status.is_client_error() {
tracing::warn!(
status = %status.as_u16(),
latency_ms = %latency_ms,
"request completed with client request error"
);
} else {
tracing::debug!(
status = %status.as_u16(),
latency_ms = %latency_ms,
"request completed"
);
}
},
),
);
let router = system_router.merge(inference_router);

// Echo x-request-id from request to response headers for client correlation
let router = router.layer(axum::middleware::from_fn(echo_request_id_header));

Ok(HttpService {
state,
Expand Down
Loading
Loading