From b318d195b6b8b2df7e698d0f37174bfab8c2d622 Mon Sep 17 00:00:00 2001 From: Michael Neale <14976+michaelneale@users.noreply.github.com> Date: Mon, 10 Aug 2026 11:15:43 +1000 Subject: [PATCH] fix(routing): stop auto from picking models nobody serves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto model selection could choose a "phantom" model — one advertised in gossip but served by no peer (stale gossip, or a peer that unloaded it) — and the request then failed with a 404 naming a model the user never asked for. The readiness filter checked local targets, then remote hosts, then fell through to `true`. A model with neither was therefore treated as ready and stayed in the auto candidate pool. It now fails closed: no routable local target and no remote host means not auto-route eligible. A freshly started serve node is kept eligible via its own hosted/serving model list, because that populates before the election target table and peer gossip catch up. Explicit model requests are unchanged and still return an honest 404. Regression introduced by the combination of #734 (added the readiness filter with the fail-open default) and #1082 (replaced the old route-to-first-available fallback with a hard 404, which made selecting a phantom user-visible). --- .../src/network/openai/auto_route.rs | 35 ++++++++++ .../src/network/openai/ingress.rs | 65 ++++++++++++++++++- 2 files changed, 99 insertions(+), 1 deletion(-) diff --git a/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs b/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs index 1a7f2d667..83d5373b2 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/auto_route.rs @@ -83,6 +83,18 @@ pub(crate) async fn model_has_eligible_remote_host( model_has_eligible_target(node, model, required_tokens, &targets, affinity).await } +/// Whether this node itself serves `model`, independent of the target table. +/// +/// A freshly started serve node loads its model and records it in +/// `serving_models` / `hosted_models` before the election target table and peer +/// gossip catch up. During that window the model has no routable target and no +/// remote host, so a strict readiness check would exclude the node's own model +/// from auto routing. This keeps it eligible. +pub(crate) async fn model_is_locally_served(node: &mesh::Node, model: &str) -> bool { + node.hosted_models().await.iter().any(|m| m == model) + || node.serving_models().await.iter().any(|m| m == model) +} + pub(crate) fn pool_for_ready_models<'a>( available: &[router::RoutingCandidate<'a>], ready_models: &[&str], @@ -132,6 +144,29 @@ mod tests { assert_eq!(pool[0].name, "ready-model"); } + #[tokio::test] + async fn model_served_by_nobody_is_not_locally_served() { + let node = mesh::Node::new_for_tests(crate::mesh::NodeRole::Worker) + .await + .expect("test node"); + + assert!(!model_is_locally_served(&node, "phantom/model:Q4_K_M").await); + } + + #[tokio::test] + async fn freshly_loaded_local_model_stays_eligible_before_targets_populate() { + let node = mesh::Node::new_for_tests(crate::mesh::NodeRole::Worker) + .await + .expect("test node"); + // A serve node records its model here before the election target table + // and peer gossip catch up. + node.set_hosted_models(vec!["local/fresh-model:Q4_K_M".to_string()]) + .await; + + assert!(model_is_locally_served(&node, "local/fresh-model:Q4_K_M").await); + assert!(!model_is_locally_served(&node, "some/other-model:Q4_K_M").await); + } + #[test] fn auto_route_pool_preserves_availability_when_none_ready() { let caps = crate::models::ModelCapabilities::default(); diff --git a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs index ec26b0d72..b298396ed 100644 --- a/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs +++ b/crates/mesh-llm-host-runtime/src/network/openai/ingress.rs @@ -352,7 +352,15 @@ async fn auto_route_model_has_ready_ingress_target( .await; } - true + // No routable local target and no peer advertises this model. Fail closed: + // such a model is a phantom (stale gossip, or a peer that unloaded it), and + // letting it stay in the pool means `auto` can pick a model that then 404s + // on a model the user never named. Explicit requests still 404 honestly. + // + // The one exception is a freshly started serve node: its own model is + // loaded and in `serving_models` before the target table and gossip catch + // up, so keep it eligible rather than excluding this node's own model. + auto_route::model_is_locally_served(node, model).await } fn maybe_enable_auto_route_hooks( @@ -1046,6 +1054,61 @@ mod tests { } } + /// A model nobody serves must not stay in the auto pool. Before this, + /// the readiness check fell through to `true`, so `auto` could select a + /// phantom (stale gossip, or a peer that unloaded) and 404 the caller on + /// a model they never named. + #[tokio::test] + async fn phantom_model_is_not_auto_route_eligible() { + let node = mesh::Node::new_for_tests(crate::mesh::NodeRole::Worker) + .await + .expect("test node"); + let targets = election::ModelTargets::default(); + let affinity = affinity::AffinityRouter::new(); + + let eligible = auto_route_model_has_ready_ingress_target( + &node, + &targets, + "phantom/model:Q4_K_M", + None, + &affinity, + ) + .await; + + assert!( + !eligible, + "a model with no local target and no remote host must not be auto-route eligible" + ); + } + + /// A freshly started serve node records its model before the target table + /// and gossip catch up. Failing closed must not exclude this node's own + /// model during that window. + #[tokio::test] + async fn freshly_served_local_model_is_auto_route_eligible() { + let node = mesh::Node::new_for_tests(crate::mesh::NodeRole::Worker) + .await + .expect("test node"); + node.set_hosted_models(vec!["local/fresh-model:Q4_K_M".to_string()]) + .await; + let targets = election::ModelTargets::default(); + let affinity = affinity::AffinityRouter::new(); + + let eligible = auto_route_model_has_ready_ingress_target( + &node, + &targets, + "local/fresh-model:Q4_K_M", + None, + &affinity, + ) + .await; + + assert!( + eligible, + "a locally served model must stay eligible before targets populate" + ); + } + #[test] fn parse_model_with_profile_with_named_profile() { let (model_ref, profile) = parse_model_with_profile("Qwen3-8B#low-ctx");