Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion clients/openapi-gen/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,8 @@ fn main() -> anyhow::Result<()> {
);

// ---- Models ----
let resp_name = collect_schema(&schema_for!(ListModelsResponse), &mut schemas)?;
use openai_protocol::models::ListModelsResponse as OpenAIListModelsResponse;
let resp_name = collect_schema(&schema_for!(OpenAIListModelsResponse), &mut schemas)?;
paths.insert(
"/v1/models".to_string(),
get_endpoint("listModels", "List available models", &resp_name),
Expand Down
1 change: 1 addition & 0 deletions crates/protocols/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ pub mod interactions;
pub mod messages;
pub mod model_card;
pub mod model_type;
pub mod models;
pub mod parser;
pub mod realtime_conversation;
pub mod realtime_events;
Expand Down
23 changes: 23 additions & 0 deletions crates/protocols/src/model_card.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};

use super::{
model_type::{Endpoint, ModelType},
models::ModelObject,
worker::ProviderType,
};

Expand Down Expand Up @@ -286,6 +287,28 @@ impl ModelCard {
self.model_type.supports_reasoning()
}

/// Get the `owned_by` string for this model.
///
/// Maps `None` → `"self_hosted"`, provider → `provider.as_str()`.
pub fn owned_by(&self) -> &str {
match &self.provider {
Some(p) => p.as_str(),
None => "self_hosted",
Comment thread
slin1237 marked this conversation as resolved.
Resolved
}
}

/// Convert this model card into an OpenAI-compatible [`ModelObject`],
/// consuming `self` to avoid cloning the model ID.
pub fn into_model_object(self) -> ModelObject {
let owned_by = self.owned_by().to_owned();
ModelObject {
id: self.id,
object: "model".to_owned(),
created: 0,
owned_by,
}
}

/// Check if this is a classification model
#[inline]
pub fn is_classifier(&self) -> bool {
Expand Down
50 changes: 50 additions & 0 deletions crates/protocols/src/models.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
//! OpenAI-compatible `/v1/models` response types.
//!
//! Provides [`ListModelsResponse`] built from in-memory [`ModelCard`] data,
//! ensuring every router returns the same format.

use std::collections::HashSet;

use serde::{Deserialize, Serialize};

use super::model_card::ModelCard;

/// A single model entry in the `/v1/models` response.
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct ModelObject {
/// Model identifier (e.g., "meta-llama/Llama-3.1-8B-Instruct").
pub id: String,
/// Always `"model"`.
pub object: String,
/// Unix timestamp of model creation (0 = unknown).
pub created: i64,
/// Who owns/hosts the model.
pub owned_by: String,
}

/// Response body for `GET /v1/models`.
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct ListModelsResponse {
/// Always `"list"`.
pub object: String,
/// Available models.
pub data: Vec<ModelObject>,
}

impl ListModelsResponse {
/// Build a response from an iterator of [`ModelCard`]s.
///
/// Deduplicates by model ID (first occurrence wins).
pub fn from_model_cards(cards: impl IntoIterator<Item = ModelCard>) -> Self {
let mut seen = HashSet::new();
let data = cards
.into_iter()
.filter(|card| seen.insert(card.id.clone()))
.map(ModelCard::into_model_object)
Comment thread
slin1237 marked this conversation as resolved.
Resolved
.collect();
Self {
object: "list".to_owned(),
data,
}
}
}
75 changes: 11 additions & 64 deletions model_gateway/src/routers/anthropic/models.rs
Original file line number Diff line number Diff line change
@@ -1,26 +1,22 @@
//! Anthropic Models API implementation

use axum::{
body::Body,
extract::Request,
http::StatusCode,
response::{IntoResponse, Response},
Json,
};
use openai_protocol::models::ListModelsResponse;
use tracing::{debug, warn};

use super::{
utils::{read_response_body_limited, should_propagate_header, ReadBodyResult},
worker::get_healthy_anthropic_workers,
AnthropicRouter,
};
use super::worker::get_healthy_anthropic_workers;
use crate::routers::error;

const MAX_RESPONSE_SIZE: usize = 10 * 1024 * 1024;

pub async fn handle_list_models(router: &AnthropicRouter, req: Request<Body>) -> Response {
pub fn handle_list_models(
router: &super::AnthropicRouter,
_req: axum::extract::Request<axum::body::Body>,
) -> Response {
debug!("Handling list models request");

let headers = req.headers();

// SECURITY: Filter by Anthropic provider in multi-provider setups
let healthy_workers = get_healthy_anthropic_workers(&router.context().worker_registry);

Expand All @@ -29,57 +25,8 @@ pub async fn handle_list_models(router: &AnthropicRouter, req: Request<Body>) ->
return error::service_unavailable("no_workers", "No healthy Anthropic workers available");
}

let worker = &healthy_workers[0];

let url = format!("{}/v1/models", worker.url());

debug!("Forwarding list models request to: {}", url);

let mut req_builder = router.http_client().get(&url);
let cards = healthy_workers.iter().flat_map(|w| w.models());
Comment thread
slin1237 marked this conversation as resolved.
Resolved
let resp = ListModelsResponse::from_model_cards(cards);
Comment thread
slin1237 marked this conversation as resolved.
Resolved

for (key, value) in headers {
if should_propagate_header(key.as_str()) {
req_builder = req_builder.header(key, value);
}
}

match req_builder.send().await {
Ok(response) => {
let status = response.status();

// SECURITY: Check content-length header first to reject obviously oversized responses
if let Some(content_length) = response.content_length() {
if content_length > MAX_RESPONSE_SIZE as u64 {
warn!(
"Response content-length too large: {} bytes (max {})",
content_length, MAX_RESPONSE_SIZE
);
return error::internal_error(
"response_too_large",
"Response body exceeds maximum size",
);
}
}

// SECURITY: Read body incrementally with size limit
match read_response_body_limited(response, MAX_RESPONSE_SIZE).await {
ReadBodyResult::Ok(body) => (status, body).into_response(),
ReadBodyResult::TooLarge => {
warn!("Response body too large (max {} bytes)", MAX_RESPONSE_SIZE);
error::internal_error(
"response_too_large",
"Response body exceeds maximum size",
)
}
ReadBodyResult::Error(e) => {
warn!("Failed to read response body: {}", e);
error::internal_error("read_error", format!("Failed to read response: {e}"))
}
}
}
Err(e) => {
warn!("Failed to forward list models request: {}", e);
error::bad_gateway("forward_failed", format!("Failed to forward request: {e}"))
}
}
(StatusCode::OK, Json(resp)).into_response()
}
2 changes: 1 addition & 1 deletion model_gateway/src/routers/anthropic/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ impl RouterTrait for AnthropicRouter {

/// Get available models from Anthropic API
async fn get_models(&self, req: Request<Body>) -> Response {
models::handle_list_models(self, req).await
models::handle_list_models(self, req)
}

fn router_type(&self) -> &'static str {
Expand Down
23 changes: 22 additions & 1 deletion model_gateway/src/routers/gemini/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,13 @@ use std::{
};

use async_trait::async_trait;
use axum::{http::HeaderMap, response::Response};
use axum::{
body::Body,
extract::Request,
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
Json,
};
use openai_protocol::interactions::InteractionsRequest;

use super::{
Expand Down Expand Up @@ -66,6 +72,21 @@ impl RouterTrait for GeminiRouter {
"gemini"
}

async fn get_models(&self, _req: Request<Body>) -> Response {
let cards = self
.shared_components
.worker_registry
.get_all()
.iter()
.flat_map(|w| w.models())
.collect::<Vec<_>>();
if cards.is_empty() {
return (StatusCode::SERVICE_UNAVAILABLE, "No models available").into_response();
}
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);
Comment thread
slin1237 marked this conversation as resolved.
Resolved
(StatusCode::OK, Json(resp)).into_response()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

async fn route_interactions(
&self,
headers: Option<&HeaderMap>,
Expand Down
19 changes: 12 additions & 7 deletions model_gateway/src/routers/http/pd_router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1238,13 +1238,18 @@ impl RouterTrait for PDRouter {
.await
}

async fn get_models(&self, req: Request<Body>) -> Response {
// Extract headers first to avoid Send issues
let headers = header_utils::copy_request_headers(&req);

// Proxy to first prefill worker
self.proxy_to_first_prefill_worker("v1/models", Some(headers))
.await
async fn get_models(&self, _req: Request<Body>) -> Response {
let cards = self
.worker_registry
.get_all()
.iter()
.flat_map(|w| w.models())
.collect::<Vec<_>>();
if cards.is_empty() {
return (StatusCode::SERVICE_UNAVAILABLE, "No models available").into_response();
}
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);
Comment thread
slin1237 marked this conversation as resolved.
Resolved
(StatusCode::OK, axum::Json(resp)).into_response()
Comment thread
slin1237 marked this conversation as resolved.
Resolved
Comment thread
slin1237 marked this conversation as resolved.
Resolved
}

async fn get_model_info(&self, req: Request<Body>) -> Response {
Expand Down
20 changes: 18 additions & 2 deletions model_gateway/src/routers/http/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -671,8 +671,24 @@ impl RouterTrait for Router {
self.proxy_get_request(req, "get_server_info").await
}

async fn get_models(&self, req: Request<Body>) -> Response {
self.proxy_get_request(req, "v1/models").await
async fn get_models(&self, _req: Request<Body>) -> Response {
let cards: Vec<_> = self
.worker_registry
.get_workers_filtered(
None,
Some(WorkerType::Regular),
Some(ConnectionMode::Http),
None,
false,
)
Comment thread
slin1237 marked this conversation as resolved.
Resolved
.iter()
.flat_map(|w| w.models())
.collect();
if cards.is_empty() {
return (StatusCode::SERVICE_UNAVAILABLE, "No models available").into_response();
}
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);
(StatusCode::OK, Json(resp)).into_response()
Comment thread
slin1237 marked this conversation as resolved.
Resolved
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

async fn get_model_info(&self, req: Request<Body>) -> Response {
Expand Down
45 changes: 3 additions & 42 deletions model_gateway/src/routers/openai/router.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use std::{
any::Any,
collections::HashSet,
sync::{atomic::AtomicBool, Arc},
time::Instant,
};
Expand Down Expand Up @@ -553,48 +552,10 @@ impl crate::routers::RouterTrait for OpenAIRouter {
let auth_header = extract_auth_header(Some(req.headers()), None);
self.refresh_external_models(auth_header.as_ref()).await;

let mut all_models = Vec::new();
let mut seen_models = HashSet::new();

for worker in &external_workers {
for model_card in worker.models() {
let owned_by = model_card
.provider
.as_ref()
.map(|p| format!("{p:?}").to_lowercase())
.unwrap_or_else(|| "unknown".to_string());

if seen_models.insert(model_card.id.clone()) {
all_models.push(json!({
"id": &model_card.id,
"object": "model",
"created": 0,
"owned_by": &owned_by,
"aliases": model_card.aliases,
"model_type": format!("{:?}", model_card.model_type),
}));
}

for alias in &model_card.aliases {
if seen_models.insert(alias.clone()) {
all_models.push(json!({
"id": alias,
"object": "model",
"created": 0,
"owned_by": &owned_by,
"primary_model": &model_card.id,
}));
}
}
}
}

let response_json = json!({
"object": "list",
"data": all_models
});
let cards = external_workers.iter().flat_map(|w| w.models());
Comment thread
slin1237 marked this conversation as resolved.
Resolved
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);

(StatusCode::OK, Json(response_json)).into_response()
(StatusCode::OK, Json(resp)).into_response()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

async fn route_chat(
Expand Down
Loading