Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion clients/openapi-gen/src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -604,7 +604,8 @@ fn main() -> anyhow::Result<()> {
);

// ---- Models ----
let resp_name = collect_schema(&schema_for!(ListModelsResponse), &mut schemas)?;
use openai_protocol::models::ListModelsResponse as OpenAIListModelsResponse;
let resp_name = collect_schema(&schema_for!(OpenAIListModelsResponse), &mut schemas)?;
paths.insert(
"/v1/models".to_string(),
get_endpoint("listModels", "List available models", &resp_name),
Expand Down
1 change: 1 addition & 0 deletions crates/protocols/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ pub mod interactions;
pub mod messages;
pub mod model_card;
pub mod model_type;
pub mod models;
pub mod parser;
pub mod realtime_conversation;
pub mod realtime_events;
Expand Down
23 changes: 23 additions & 0 deletions crates/protocols/src/model_card.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use serde::{Deserialize, Serialize};

use super::{
model_type::{Endpoint, ModelType},
models::ModelObject,
worker::ProviderType,
};

Expand Down Expand Up @@ -286,6 +287,28 @@ impl ModelCard {
self.model_type.supports_reasoning()
}

/// Get the `owned_by` string for this model.
///
/// Maps `None` → `"self_hosted"`, provider → `provider.as_str()`.
pub fn owned_by(&self) -> &str {
match &self.provider {
Some(p) => p.as_str(),
None => "self_hosted",
Comment thread
slin1237 marked this conversation as resolved.
Resolved
}
}

/// Convert this model card into an OpenAI-compatible [`ModelObject`],
/// consuming `self` to avoid cloning the model ID.
pub fn into_model_object(self) -> ModelObject {
let owned_by = self.owned_by().to_owned();
ModelObject {
id: self.id,
object: "model".to_owned(),
created: 0,
owned_by,
}
}

/// Check if this is a classification model
#[inline]
pub fn is_classifier(&self) -> bool {
Expand Down
50 changes: 50 additions & 0 deletions crates/protocols/src/models.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
//! OpenAI-compatible `/v1/models` response types.
//!
//! Provides [`ListModelsResponse`] built from in-memory [`ModelCard`] data,
//! ensuring every router returns the same format.

use std::collections::HashSet;

use serde::{Deserialize, Serialize};

use super::model_card::ModelCard;

/// A single model entry in the `/v1/models` response.
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct ModelObject {
/// Model identifier (e.g., "meta-llama/Llama-3.1-8B-Instruct").
pub id: String,
/// Always `"model"`.
pub object: String,
/// Unix timestamp of model creation (0 = unknown).
pub created: i64,
/// Who owns/hosts the model.
pub owned_by: String,
}

/// Response body for `GET /v1/models`.
#[derive(Debug, Clone, Serialize, Deserialize, schemars::JsonSchema)]
pub struct ListModelsResponse {
/// Always `"list"`.
pub object: String,
/// Available models.
pub data: Vec<ModelObject>,
}

impl ListModelsResponse {
/// Build a response from an iterator of [`ModelCard`]s.
///
/// Deduplicates by model ID (first occurrence wins).
pub fn from_model_cards(cards: impl IntoIterator<Item = ModelCard>) -> Self {
let mut seen = HashSet::new();
let data = cards
.into_iter()
.filter(|card| seen.insert(card.id.clone()))
.map(ModelCard::into_model_object)
Comment thread
slin1237 marked this conversation as resolved.
Resolved
.collect();
Self {
object: "list".to_owned(),
data,
}
}
}
75 changes: 11 additions & 64 deletions model_gateway/src/routers/anthropic/models.rs
Original file line number Diff line number Diff line change
@@ -1,26 +1,22 @@
//! Anthropic Models API implementation

use axum::{
body::Body,
extract::Request,
http::StatusCode,
response::{IntoResponse, Response},
Json,
};
use openai_protocol::models::ListModelsResponse;
use tracing::{debug, warn};

use super::{
utils::{read_response_body_limited, should_propagate_header, ReadBodyResult},
worker::get_healthy_anthropic_workers,
AnthropicRouter,
};
use super::worker::get_healthy_anthropic_workers;
use crate::routers::error;

const MAX_RESPONSE_SIZE: usize = 10 * 1024 * 1024;

pub async fn handle_list_models(router: &AnthropicRouter, req: Request<Body>) -> Response {
pub fn handle_list_models(
router: &super::AnthropicRouter,
_req: axum::extract::Request<axum::body::Body>,
) -> Response {
debug!("Handling list models request");

let headers = req.headers();

// SECURITY: Filter by Anthropic provider in multi-provider setups
let healthy_workers = get_healthy_anthropic_workers(&router.context().worker_registry);

Expand All @@ -29,57 +25,8 @@ pub async fn handle_list_models(router: &AnthropicRouter, req: Request<Body>) ->
return error::service_unavailable("no_workers", "No healthy Anthropic workers available");
}

let worker = &healthy_workers[0];

let url = format!("{}/v1/models", worker.url());

debug!("Forwarding list models request to: {}", url);

let mut req_builder = router.http_client().get(&url);
let cards = healthy_workers.iter().flat_map(|w| w.models());
Comment thread
slin1237 marked this conversation as resolved.
Resolved
let resp = ListModelsResponse::from_model_cards(cards);
Comment thread
slin1237 marked this conversation as resolved.
Resolved

for (key, value) in headers {
if should_propagate_header(key.as_str()) {
req_builder = req_builder.header(key, value);
}
}

match req_builder.send().await {
Ok(response) => {
let status = response.status();

// SECURITY: Check content-length header first to reject obviously oversized responses
if let Some(content_length) = response.content_length() {
if content_length > MAX_RESPONSE_SIZE as u64 {
warn!(
"Response content-length too large: {} bytes (max {})",
content_length, MAX_RESPONSE_SIZE
);
return error::internal_error(
"response_too_large",
"Response body exceeds maximum size",
);
}
}

// SECURITY: Read body incrementally with size limit
match read_response_body_limited(response, MAX_RESPONSE_SIZE).await {
ReadBodyResult::Ok(body) => (status, body).into_response(),
ReadBodyResult::TooLarge => {
warn!("Response body too large (max {} bytes)", MAX_RESPONSE_SIZE);
error::internal_error(
"response_too_large",
"Response body exceeds maximum size",
)
}
ReadBodyResult::Error(e) => {
warn!("Failed to read response body: {}", e);
error::internal_error("read_error", format!("Failed to read response: {e}"))
}
}
}
Err(e) => {
warn!("Failed to forward list models request: {}", e);
error::bad_gateway("forward_failed", format!("Failed to forward request: {e}"))
}
}
(StatusCode::OK, Json(resp)).into_response()
}
2 changes: 1 addition & 1 deletion model_gateway/src/routers/anthropic/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ impl RouterTrait for AnthropicRouter {

/// Get available models from Anthropic API
async fn get_models(&self, req: Request<Body>) -> Response {
models::handle_list_models(self, req).await
models::handle_list_models(self, req)
}

fn router_type(&self) -> &'static str {
Expand Down
20 changes: 19 additions & 1 deletion model_gateway/src/routers/gemini/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,13 @@ use std::{
};

use async_trait::async_trait;
use axum::{http::HeaderMap, response::Response};
use axum::{
body::Body,
extract::Request,
http::{HeaderMap, StatusCode},
response::{IntoResponse, Response},
Json,
};
use openai_protocol::interactions::InteractionsRequest;

use super::{
Expand Down Expand Up @@ -66,6 +72,18 @@ impl RouterTrait for GeminiRouter {
"gemini"
}

async fn get_models(&self, _req: Request<Body>) -> Response {
let cards = self
.shared_components
.worker_registry
.get_all()
.iter()
.flat_map(|w| w.models())
.collect::<Vec<_>>();
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);
Comment thread
slin1237 marked this conversation as resolved.
Resolved
(StatusCode::OK, Json(resp)).into_response()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

async fn route_interactions(
&self,
headers: Option<&HeaderMap>,
Expand Down
16 changes: 9 additions & 7 deletions model_gateway/src/routers/http/pd_router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1238,13 +1238,15 @@ impl RouterTrait for PDRouter {
.await
}

async fn get_models(&self, req: Request<Body>) -> Response {
// Extract headers first to avoid Send issues
let headers = header_utils::copy_request_headers(&req);

// Proxy to first prefill worker
self.proxy_to_first_prefill_worker("v1/models", Some(headers))
.await
async fn get_models(&self, _req: Request<Body>) -> Response {
let cards = self
.worker_registry
.get_all()
.iter()
.flat_map(|w| w.models())
.collect::<Vec<_>>();
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);
Comment thread
slin1237 marked this conversation as resolved.
Resolved
(StatusCode::OK, axum::Json(resp)).into_response()
Comment thread
slin1237 marked this conversation as resolved.
Resolved
Comment thread
slin1237 marked this conversation as resolved.
Resolved
}

async fn get_model_info(&self, req: Request<Body>) -> Response {
Expand Down
11 changes: 9 additions & 2 deletions model_gateway/src/routers/http/router.rs
Original file line number Diff line number Diff line change
Expand Up @@ -671,8 +671,15 @@ impl RouterTrait for Router {
self.proxy_get_request(req, "get_server_info").await
}

async fn get_models(&self, req: Request<Body>) -> Response {
self.proxy_get_request(req, "v1/models").await
async fn get_models(&self, _req: Request<Body>) -> Response {
let cards = self
.worker_registry
.get_all()
.iter()
.flat_map(|w| w.models())
.collect::<Vec<_>>();
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);
Comment thread
slin1237 marked this conversation as resolved.
Resolved
(StatusCode::OK, Json(resp)).into_response()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
Comment thread
slin1237 marked this conversation as resolved.
Resolved
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

async fn get_model_info(&self, req: Request<Body>) -> Response {
Expand Down
45 changes: 3 additions & 42 deletions model_gateway/src/routers/openai/router.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use std::{
any::Any,
collections::HashSet,
sync::{atomic::AtomicBool, Arc},
time::Instant,
};
Expand Down Expand Up @@ -553,48 +552,10 @@ impl crate::routers::RouterTrait for OpenAIRouter {
let auth_header = extract_auth_header(Some(req.headers()), None);
self.refresh_external_models(auth_header.as_ref()).await;

let mut all_models = Vec::new();
let mut seen_models = HashSet::new();

for worker in &external_workers {
for model_card in worker.models() {
let owned_by = model_card
.provider
.as_ref()
.map(|p| format!("{p:?}").to_lowercase())
.unwrap_or_else(|| "unknown".to_string());

if seen_models.insert(model_card.id.clone()) {
all_models.push(json!({
"id": &model_card.id,
"object": "model",
"created": 0,
"owned_by": &owned_by,
"aliases": model_card.aliases,
"model_type": format!("{:?}", model_card.model_type),
}));
}

for alias in &model_card.aliases {
if seen_models.insert(alias.clone()) {
all_models.push(json!({
"id": alias,
"object": "model",
"created": 0,
"owned_by": &owned_by,
"primary_model": &model_card.id,
}));
}
}
}
}

let response_json = json!({
"object": "list",
"data": all_models
});
let cards = external_workers.iter().flat_map(|w| w.models());
Comment thread
slin1237 marked this conversation as resolved.
Resolved
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);

(StatusCode::OK, Json(response_json)).into_response()
(StatusCode::OK, Json(resp)).into_response()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

async fn route_chat(
Expand Down
34 changes: 9 additions & 25 deletions model_gateway/src/routers/router_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ use openai_protocol::{
rerank::RerankRequest,
responses::{ResponsesGetParams, ResponsesRequest},
};
use serde_json::Value;
use tracing::{debug, info, warn};

use crate::{
Expand Down Expand Up @@ -404,32 +403,17 @@ impl RouterTrait for RouterManager {
}

// Fallback: return OpenAI-compatible format from worker registry
let model_names = self.worker_registry.get_models();

if model_names.is_empty() {
let cards = self
.worker_registry
.get_all()
.iter()
.flat_map(|w| w.models())
.collect::<Vec<_>>();
if cards.is_empty() {
(StatusCode::SERVICE_UNAVAILABLE, "No models available").into_response()
} else {
// Convert model names to OpenAI-compatible model objects
let models: Vec<Value> = model_names
.iter()
.map(|name| {
serde_json::json!({
"id": name,
"object": "model",
"owned_by": "local"
})
})
.collect();

(
StatusCode::OK,
serde_json::json!({
"object": "list",
"data": models
})
.to_string(),
)
.into_response()
let resp = openai_protocol::models::ListModelsResponse::from_model_cards(cards);
(StatusCode::OK, axum::Json(resp)).into_response()
}
Comment thread
slin1237 marked this conversation as resolved.
Resolved
}

Expand Down
Loading
Loading