Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions lib/llm/src/entrypoint/input/http.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,25 @@ impl HttpFrontend {
anyhow::bail!("custom router plugins require a dynamic engine");
}

// Hosts that pass no resolved plugins resolve the policy file later, so read its
// selection from the config as well.
let router_config = engine_config.local_model().router_config();
if router_config.router_mode.is_kv_routing()
&& !self.plugins.has_custom_worker_selection()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
&& matches!(
router_config
.kv_router_config
.selected_worker_selection_policy_instance(),
Ok(None)
)
{
tracing::warn!(
"Starting with Dynamo 1.8, --router-mode kv will default to the two-tier cost \
function (dynamo-two-tier-cost-fn). To keep the current cost function, select a \
dynamo-default-cost-fn instance in --router-policy-config."
);
}

let plugins = RouterPluginBuilder::new(self.plugins);
plugins.validate_config(&engine_config.local_model().router_config().kv_router_config)?;

Expand Down
17 changes: 12 additions & 5 deletions lib/llm/src/kv_router/plugins.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,33 @@

use std::sync::OnceLock;

use dynamo_kv_router::plugins::worker_selection::WorkerSelectionPolicyFactory;
use dynamo_kv_router::plugins::{RouterPluginRegistry, RouterPlugins};

use super::{KvRouter, SelectionPolicySource};

static INSTALLED_PLUGINS: OnceLock<RouterPluginRegistry> = OnceLock::new();

/// The worker-selection policy hosts run when no policy is configured, including for a
/// `default` selection. Every host install path reads it from here, so changing the default
/// policy is a change to this function alone.
fn host_default_factory() -> WorkerSelectionPolicyFactory {
dynamo_custom_policy_builtin::default_factory()
}

/// Install the linked catalog once for frontend and embedded router construction.
/// Returns `false` if a catalog has already been installed.
pub fn install_router_plugin_registry(registry: RouterPluginRegistry) -> bool {
INSTALLED_PLUGINS
.set(registry.with_default_factory(dynamo_custom_policy_builtin::default_factory()))
.set(registry.with_default_factory(host_default_factory()))
.is_ok()
}

/// The installed catalog, or the required builtin default policy.
pub fn router_plugin_registry() -> RouterPluginRegistry {
INSTALLED_PLUGINS
.get()
.cloned()
.unwrap_or_else(dynamo_custom_policy_builtin::default_registry)
INSTALLED_PLUGINS.get().cloned().unwrap_or_else(|| {
RouterPluginRegistry::default().with_default_factory(host_default_factory())
})
}

/// Carries configured plugins through the host's shared router construction.
Expand Down
Loading