Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 67 additions & 0 deletions model_gateway/src/routers/factory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,31 @@ use crate::{
policies::PolicyFactory,
};

#[derive(Debug, Clone, Hash, Eq, PartialEq)]
pub struct RouterId(&'static str);

impl RouterId {
pub const fn new(id: &'static str) -> Self {
Self(id)
}

pub fn as_str(&self) -> &str {
self.0
}
}
Comment on lines +22 to +30

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

To make RouterId more ergonomic, consider implementing the Display and Deref traits. This would allow you to use it directly in formatting macros (like info!) and other places expecting a &str, removing the need for the .as_str() method.

After applying this suggestion, you can remove the .as_str() calls in router_manager.rs.

impl RouterId {
    pub const fn new(id: &'static str) -> Self {
        Self(id)
    }
}

impl std::fmt::Display for RouterId {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        write!(f, "{}", self.0)
    }
}

impl std::ops::Deref for RouterId {
    type Target = str;

    fn deref(&self) -> &Self::Target {
        self.0
    }
}


/// Static router ID constants to avoid heap allocations in hot paths
pub mod router_ids {
use super::RouterId;

pub const HTTP_REGULAR: RouterId = RouterId::new("http-regular");
pub const HTTP_PD: RouterId = RouterId::new("http-pd");
pub const HTTP_OPENAI: RouterId = RouterId::new("http-openai");
pub const HTTP_ANTHROPIC: RouterId = RouterId::new("http-anthropic");
pub const GRPC_REGULAR: RouterId = RouterId::new("grpc-regular");
pub const GRPC_PD: RouterId = RouterId::new("grpc-pd");
}

/// Factory for creating router instances based on configuration
pub struct RouterFactory;

Expand Down Expand Up @@ -143,4 +168,46 @@ impl RouterFactory {
let router = AnthropicRouter::new(ctx.clone());
Ok(Box::new(router))
}

/// Create all routers for IGW (multi-router) mode.
///
/// Returns a list of (router_id, label, creation_result) tuples.
/// Adding a new router to IGW mode only requires adding a line here.
pub async fn create_igw_routers(
policy: &PolicyConfig,
ctx: &Arc<AppContext>,
) -> Vec<(RouterId, &'static str, Result<Box<dyn RouterTrait>, String>)> {
vec![
(
router_ids::HTTP_REGULAR,
"HTTP Regular",
Self::create_regular_router(ctx).await,
),
(
router_ids::GRPC_REGULAR,
"gRPC Regular",
Self::create_grpc_router(ctx).await,
),
(
router_ids::HTTP_PD,
"HTTP PD",
Self::create_pd_router(None, None, policy, ctx).await,
),
(
router_ids::GRPC_PD,
"gRPC PD",
Self::create_grpc_pd_router(None, None, policy, ctx).await,
),
(
router_ids::HTTP_OPENAI,
"OpenAI",
Self::create_openai_router(ctx).await,
),
(
router_ids::HTTP_ANTHROPIC,
"Anthropic",
Self::create_anthropic_router(ctx).await,
),
]
}
}
121 changes: 26 additions & 95 deletions model_gateway/src/routers/router_manager.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,35 +32,13 @@ use crate::{
rerank::RerankRequest,
responses::{ResponsesGetParams, ResponsesRequest},
},
routers::RouterTrait,
routers::{
factory::{router_ids, RouterId},
RouterFactory, RouterTrait,
},
server::ServerConfig,
};

#[derive(Debug, Clone, Hash, Eq, PartialEq)]
pub struct RouterId(&'static str);

impl RouterId {
pub const fn new(id: &'static str) -> Self {
Self(id)
}

pub fn as_str(&self) -> &str {
self.0
}
}

/// Static router ID constants to avoid heap allocations in hot paths
pub mod router_ids {
use super::RouterId;

pub const HTTP_REGULAR: RouterId = RouterId::new("http-regular");
pub const HTTP_PD: RouterId = RouterId::new("http-pd");
pub const HTTP_OPENAI: RouterId = RouterId::new("http-openai");
pub const HTTP_ANTHROPIC: RouterId = RouterId::new("http-anthropic");
pub const GRPC_REGULAR: RouterId = RouterId::new("grpc-regular");
pub const GRPC_PD: RouterId = RouterId::new("grpc-pd");
}

pub struct RouterManager {
worker_registry: Arc<WorkerRegistry>,
routers: Arc<DashMap<RouterId, Arc<dyn RouterTrait>>>,
Expand All @@ -80,87 +58,40 @@ impl RouterManager {
}
}

/// Register a router if creation succeeded, log either way.
fn try_register(
&self,
id: RouterId,
label: &str,
result: Result<Box<dyn RouterTrait>, String>,
) {
match result {
Ok(router) => {
info!("Created {label} router");
self.register_router(id, Arc::from(router));
}
Err(e) => {
warn!("Failed to create {label} router: {e}");
}
}
}

pub async fn from_config(
config: &ServerConfig,
app_context: &Arc<AppContext>,
) -> Result<Arc<Self>, String> {
use crate::routers::RouterFactory;

let mut manager = Self::new(app_context.worker_registry.clone());
manager.enable_igw = config.router_config.enable_igw;
let manager = Arc::new(manager);

if config.router_config.enable_igw {
info!("Initializing RouterManager in multi-router mode (IGW)");

match RouterFactory::create_regular_router(app_context).await {
Ok(http_regular) => {
info!("Created HTTP Regular router");
manager.register_router(router_ids::HTTP_REGULAR, Arc::from(http_regular));
}
Err(e) => {
warn!("Failed to create HTTP Regular router: {e}");
}
}

// Always create gRPC Regular router in IGW mode
match RouterFactory::create_grpc_router(app_context).await {
Ok(grpc_regular) => {
info!("Created gRPC Regular router");
manager.register_router(router_ids::GRPC_REGULAR, Arc::from(grpc_regular));
}
Err(e) => {
warn!("Failed to create gRPC Regular router: {e}");
}
}

info!("PD disaggregation auto-enabled for IGW mode, creating PD routers");

// Create HTTP PD router
match RouterFactory::create_pd_router(
None,
None,
&config.router_config.policy,
app_context,
)
.await
{
Ok(http_pd) => {
info!("Created HTTP PD router");
manager.register_router(router_ids::HTTP_PD, Arc::from(http_pd));
}
Err(e) => {
warn!("Failed to create HTTP PD router: {e}");
}
}
let routers =
RouterFactory::create_igw_routers(&config.router_config.policy, app_context).await;

// Create gRPC PD router
match RouterFactory::create_grpc_pd_router(
None,
None,
&config.router_config.policy,
app_context,
)
.await
{
Ok(grpc_pd) => {
info!("Created gRPC PD router");
manager.register_router(router_ids::GRPC_PD, Arc::from(grpc_pd));
}
Err(e) => {
warn!("Failed to create gRPC PD router: {e}");
}
}

// Create OpenAI router for external OpenAI-compatible backends
match RouterFactory::create_openai_router(app_context).await {
Ok(openai) => {
info!("Created OpenAI router");
manager.register_router(router_ids::HTTP_OPENAI, Arc::from(openai));
}
Err(e) => {
warn!("Failed to create OpenAI router: {e}");
}
for (id, label, result) in routers {
manager.try_register(id, label, result);
}

info!(
Expand Down
Loading