Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
284 changes: 284 additions & 0 deletions model_gateway/src/routers/openai/chat.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,284 @@
//! Chat completion routing for the OpenAI router.

use std::{sync::Arc, time::Instant};

use axum::{
body::Body,
http::{header::CONTENT_TYPE, HeaderMap, HeaderValue, StatusCode},
response::Response,
};
use futures_util::StreamExt;
use openai_protocol::chat::ChatCompletionRequest;
use serde_json::to_value;
use tokio::sync::mpsc;
use tokio_stream::wrappers::UnboundedReceiverStream;

use super::{
context::{ComponentRefs, PayloadState, RequestContext, SharedComponents, WorkerSelection},
provider::ProviderRegistry,
router::resolve_provider,
};
use crate::{
config::types::RetryConfig,
core::{is_retryable_status, Endpoint, ProviderType, RetryExecutor, WorkerRegistry},
observability::metrics::{bool_to_static_str, metrics_labels, Metrics},
routers::{
error,
header_utils::{apply_provider_headers, extract_auth_header},
worker_selection::{SelectWorkerRequest, WorkerSelector},
},
};

/// Shared context passed to chat routing functions.
pub(super) struct RouterContext<'a> {
pub worker_registry: &'a WorkerRegistry,
pub provider_registry: &'a ProviderRegistry,
pub shared_components: &'a Arc<SharedComponents>,
pub client: &'a reqwest::Client,
pub retry_config: &'a RetryConfig,
}

/// Route a chat completion request to the appropriate upstream worker.
pub(super) async fn route_chat(
deps: &RouterContext<'_>,
headers: Option<&HeaderMap>,
body: &ChatCompletionRequest,
model_id: Option<&str>,
) -> Response {
let start = Instant::now();
let model = model_id.unwrap_or(body.model.as_str());
let streaming = body.stream;
Comment on lines +48 to +50

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Rewrite the chat request with the override model before routing it.

The function resolves model from model_id, but Line 64 still selects on body.model and Line 85 serializes the original body unchanged. If model_id is set, metrics/provider resolution point at one model while worker lookup and the upstream payload use another.

🐛 Proposed fix
-    let start = Instant::now();
-    let model = model_id.unwrap_or(body.model.as_str());
+    let start = Instant::now();
+    let mut request_body = body.clone();
+    if let Some(model_id) = model_id {
+        request_body.model = model_id.to_string();
+    }
+    let model = request_body.model.as_str();
     let streaming = body.stream;
@@
         .select_worker(&SelectWorkerRequest {
-            model_id: body.model.as_str(),
+            model_id: request_body.model.as_str(),
             headers,
             provider: Some(ProviderType::OpenAI),
             ..Default::default()
         })
@@
-    let mut payload = match to_value(body) {
+    let mut payload = match to_value(&request_body) {
         Ok(v) => v,

I'd also pass request_body.clone() into RequestContext::for_chat so downstream state stays aligned.

Also applies to: 61-68, 85-120

🤖 Prompt for AI Agents
Verify each finding against the current code and only fix it if needed.

In `@model_gateway/src/routers/openai/chat.rs` around lines 48 - 50, The code
resolves an override via model_id into the local variable model but never
rewrites the outgoing chat request or the RequestContext, causing mismatched
metrics/provider selection vs the upstream payload; update the chat path to
mutate or build a new request_body where body.model is set to model (i.e., apply
the model_id override into the serialized payload used for routing/worker
lookup) before any provider resolution and before serializing the body, and pass
request_body.clone() into RequestContext::for_chat so downstream state and
metrics use the same overridden model; ensure all branches that inspect
body.model or serialize body (the selection logic around model usage and the
serialization near where streaming/worker lookup occurs) use this rewritten
request_body.


Metrics::record_router_request(
metrics_labels::ROUTER_OPENAI,
metrics_labels::BACKEND_EXTERNAL,
metrics_labels::CONNECTION_HTTP,
model,
metrics_labels::ENDPOINT_CHAT,
bool_to_static_str(streaming),
);
Comment thread
slin1237 marked this conversation as resolved.
Resolved

let selector = WorkerSelector::new(deps.worker_registry, deps.client);
let worker = match selector
.select_worker(&SelectWorkerRequest {
model_id: model,
headers,
provider: Some(ProviderType::OpenAI),
..Default::default()
})
.await
{
Ok(w) => w,
Err(response) => {
Metrics::record_router_error(
metrics_labels::ROUTER_OPENAI,
metrics_labels::BACKEND_EXTERNAL,
metrics_labels::CONNECTION_HTTP,
model,
metrics_labels::ENDPOINT_CHAT,
metrics_labels::ERROR_NO_WORKERS,
);
return response;
}
};

let mut payload = match to_value(body) {
Ok(v) => v,
Err(e) => {
Metrics::record_router_error(
metrics_labels::ROUTER_OPENAI,
metrics_labels::BACKEND_EXTERNAL,
metrics_labels::CONNECTION_HTTP,
model,
metrics_labels::ENDPOINT_CHAT,
metrics_labels::ERROR_VALIDATION,
);
return error::bad_request(
"invalid_request",
format!("Failed to serialize request: {e}"),
);
}
};

// When model_id overrides the body model, patch the serialized payload
// so the upstream request uses the effective model consistently.
if model_id.is_some() {
payload["model"] = serde_json::Value::String(model.to_owned());
}

let provider = resolve_provider(deps.provider_registry, worker.as_ref(), model);
if let Err(e) = provider.transform_request(&mut payload, Endpoint::Chat) {
Metrics::record_router_error(
metrics_labels::ROUTER_OPENAI,
metrics_labels::BACKEND_EXTERNAL,
metrics_labels::CONNECTION_HTTP,
model,
metrics_labels::ENDPOINT_CHAT,
metrics_labels::ERROR_VALIDATION,
);
return error::bad_request("invalid_request", format!("Provider transform error: {e}"));
}

let mut ctx = RequestContext::for_chat(
Arc::new(body.clone()),
headers.cloned(),
model_id.map(String::from),
ComponentRefs::Shared(Arc::clone(deps.shared_components)),
);

ctx.state.worker = Some(WorkerSelection {
worker: Arc::clone(&worker),
provider,
});

let url = format!("{}/v1/chat/completions", worker.url());
ctx.state.payload = Some(PayloadState {
json: payload,
url: url.clone(),
previous_response_id: None,
});

// Wrap values in Arc to avoid cloning large objects on each retry attempt
#[expect(
clippy::expect_used,
reason = "payload is set earlier in this function; absence is a logic error"
)]
let payload_ref = ctx.payload().expect("Payload not prepared");
let payload_json = Arc::new(payload_ref.json.clone());
let client = ctx.components.client().clone();
let headers_cloned = Arc::new(ctx.headers().cloned());
let worker_api_key = Arc::new(worker.api_key().cloned());
let is_streaming = ctx.is_streaming();

let response = RetryExecutor::execute_response_with_retry(
deps.retry_config,
|_attempt| {
let client = client.clone();
let url = url.clone();
let payload = Arc::clone(&payload_json);
let headers = Arc::clone(&headers_cloned);
let worker_api_key = Arc::clone(&worker_api_key);
let worker = Arc::clone(&worker);

async move {
let mut req = client.post(&url).json(&*payload);
let auth_header =
extract_auth_header((*headers).as_ref(), (*worker_api_key).as_ref());
req = apply_provider_headers(req, &url, auth_header.as_ref());

if is_streaming {
req = req.header("Accept", "text/event-stream");
}

let resp = match req.send().await {
Ok(r) => r,
Err(e) => {
worker.circuit_breaker().record_failure();
return error::service_unavailable(
"upstream_error",
format!("Failed to contact upstream: {e}"),
);
}
};

let status = StatusCode::from_u16(resp.status().as_u16())
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR);

if !status.is_success() {
worker.circuit_breaker().record_failure();
}

if is_streaming {
if status.is_success() {
worker.circuit_breaker().record_success();
}
let stream = resp.bytes_stream();
let (tx, rx) = mpsc::unbounded_channel();
#[expect(clippy::disallowed_methods, reason = "fire-and-forget stream relay; gateway shutdown need not wait for individual stream forwarding")]
tokio::spawn(async move {
let mut s = stream;
while let Some(chunk) = s.next().await {
match chunk {
Ok(bytes) => {
if tx.send(Ok(bytes)).is_err() {
break;
}
}
Err(e) => {
let _ = tx.send(Err(format!("Stream error: {e}")));
break;
}
}
}
});
let mut response =
Response::new(Body::from_stream(UnboundedReceiverStream::new(rx)));
*response.status_mut() = status;
response
.headers_mut()
.insert(CONTENT_TYPE, HeaderValue::from_static("text/event-stream"));
response
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
} else {
let content_type = resp.headers().get(CONTENT_TYPE).cloned();
match resp.bytes().await {
Ok(body) => {
if status.is_success() {
worker.circuit_breaker().record_success();
}
let mut response = Response::new(Body::from(body));
*response.status_mut() = status;
if let Some(ct) = content_type {
response.headers_mut().insert(CONTENT_TYPE, ct);
}
response
}
Err(e) => {
worker.circuit_breaker().record_failure();
error::internal_error(
"upstream_error",
format!("Failed to read response: {e}"),
)
}
}
}
}
},
|res, _attempt| is_retryable_status(res.status()),
|delay, attempt| {
Metrics::record_worker_retry(
metrics_labels::BACKEND_EXTERNAL,
metrics_labels::ENDPOINT_CHAT,
);
Metrics::record_worker_retry_backoff(attempt, delay);
},
|| {
Metrics::record_worker_retries_exhausted(
metrics_labels::BACKEND_EXTERNAL,
metrics_labels::ENDPOINT_CHAT,
);
},
)
.await;

if response.status().is_success() {
Metrics::record_router_duration(
metrics_labels::ROUTER_OPENAI,
metrics_labels::BACKEND_EXTERNAL,
metrics_labels::CONNECTION_HTTP,
model,
metrics_labels::ENDPOINT_CHAT,
start.elapsed(),
);
} else {
Metrics::record_router_error(
metrics_labels::ROUTER_OPENAI,
metrics_labels::BACKEND_EXTERNAL,
metrics_labels::CONNECTION_HTTP,
model,
metrics_labels::ENDPOINT_CHAT,
metrics_labels::ERROR_BACKEND,
);
}

response
}
70 changes: 70 additions & 0 deletions model_gateway/src/routers/openai/health.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
//! Health check and server info endpoints for the OpenAI router.

use std::sync::Arc;

use axum::{
http::StatusCode,
response::{IntoResponse, Response},
Json,
};
use serde_json::json;

use crate::{
core::{RuntimeType, Worker, WorkerRegistry},
routers::error,
};

fn external_workers(registry: &WorkerRegistry) -> Vec<Arc<dyn Worker>> {
registry
.get_all()
.into_iter()
.filter(|w| w.metadata().spec.runtime_type == RuntimeType::External)
.collect()
}

pub(super) fn health_generate(registry: &WorkerRegistry) -> Response {
let workers = external_workers(registry);
if workers.is_empty() {
return error::service_unavailable("service_unavailable", "No external workers registered");
}

let (healthy, unhealthy): (Vec<_>, Vec<_>) = workers.iter().partition(|w| w.is_healthy());

if unhealthy.is_empty() {
(
StatusCode::OK,
format!("OK - {} workers healthy", healthy.len()),
)
.into_response()
} else {
let unhealthy_info: Vec<_> = unhealthy
.iter()
.map(|w| format!("{} ({})", w.model_id(), w.url()))
Comment thread
slin1237 marked this conversation as resolved.
Resolved
.collect();
error::service_unavailable(
"service_unavailable",
format!(
"{}/{} workers unhealthy: {}",
unhealthy.len(),
workers.len(),
unhealthy_info.join(", ")
),
)
}
}

pub(super) fn get_server_info(registry: &WorkerRegistry) -> Response {
let stats = registry.stats();
let workers = external_workers(registry);
let worker_urls: Vec<_> = workers.iter().map(|w| w.url()).collect();

let info = json!({
"router_type": "openai",
"total_workers": stats.total_workers,
"external_workers": workers.len(),
"healthy_workers": stats.healthy_workers,
"total_models": stats.total_models,
"worker_urls": worker_urls
Comment thread
slin1237 marked this conversation as resolved.
Resolved
});
(StatusCode::OK, Json(info)).into_response()
}
2 changes: 2 additions & 0 deletions model_gateway/src/routers/openai/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@
//! - Multi-turn tool execution loops
//! - SSE (Server-Sent Events) streaming

mod chat;
mod context;
mod health;
mod provider;
pub mod realtime;
pub mod responses;
Expand Down
Loading
Loading