From 5b09baa6f4ec20ad417b77ca67db664094027edc Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Tue, 6 Oct 2026 13:42:41 +0000
Subject: [PATCH 1/4] fix(cache_aware): gate the miss path and ignore idle
 throughput gauges

A request at or below cache_threshold went straight to the expected-wait
score over every healthy worker: a miss was "not a spill", so it skipped
the count-pressure gate every other path applies. The score keeps no
count bound of its own, and its rate term divides by gen_throughput,
which an engine never clears: an idle SGLang worker reports the ~0.35
tok/s left by its last one-token health generation, so one dispatch read
as thousands of seconds and a burst of unrelated prompts piled onto the
other worker until the next poll (2:30 on mock workers, 29:3 on real
ones, with free slots on the loser the whole time).

Route a miss, like a spill, among the healthy workers under the gate, on
the tree, event-driven and hash paths; the KV-pressure fallback stays
ungated, since a count gate there could steer onto a KV-saturated
engine. Score against the rate summed over ranks with requests running,
so an idle rank's stale gauge counts for nothing and the worker scores at
the default throughput, like one that reports no rate.

Closes #2804
---
 model_gateway/src/policies/cache_aware.rs  | 207 ++++++++++++++++++---
 model_gateway/src/policies/least_load.rs   |  86 ++++++++-
 model_gateway/src/policies/mod.rs          |   4 +-
 model_gateway/src/policies/power_of_two.rs |   4 +-
 4 files changed, 269 insertions(+), 32 deletions(-)

diff --git a/model_gateway/src/policies/cache_aware.rs b/model_gateway/src/policies/cache_aware.rs
index 9f20d8ce55..e2e1d831b3 100644
--- a/model_gateway/src/policies/cache_aware.rs
+++ b/model_gateway/src/policies/cache_aware.rs
@@ -1333,7 +1333,10 @@ impl CacheAwarePolicy {
     /// hot prefixes replicate instead of queueing behind one engine. Both
     /// margins must clear so the gate neither fires on steady-state variance
     /// (relative alone would, at low means) nor stays blind to a deep queue
-    /// (absolute alone would, at high means).
+    /// (absolute alone would, at high means). A request with no holder is
+    /// routed among the workers under the same gate
+    /// ([`Self::ungated_candidates`]), so a burst of misses is bounded by
+    /// the margins a hit spills at.
     fn candidate_requires_spill(
         &self,
         workers: &[Arc<dyn Worker>],
@@ -1367,23 +1370,36 @@ impl CacheAwarePolicy {
             return self.select_expected_wait(workers, &safe_affinity, info);
         }
 
-        // A miss has no affinity candidates and is not a spill: every healthy
-        // worker participates. A true spill excludes other workers that trip
-        // the same gate, preventing a fallback from reselecting a hot holder.
-        if affinity_candidates.is_empty() {
-            return self.select_expected_wait(workers, healthy_indices, info);
-        }
-        let spill_candidates: Vec<usize> = healthy_indices
+        // No safe holder: a miss (no holder at all) or a spill (every holder
+        // gated). Both go to the expected-wait worker among those under the
+        // same gate, so a spill never reselects a hot holder and a burst of
+        // misses is bounded by the margins a hit spills at. Expected wait
+        // alone keeps no count bound: its queue and rate signals are a poll
+        // old, and a burst routed on them piles onto one worker while
+        // another has free slots.
+        let candidates = self.ungated_candidates(workers, healthy_indices, avg_load);
+        self.select_expected_wait(workers, &candidates, info)
+    }
+
+    /// The healthy workers under the count-pressure gate. Every worker
+    /// trips it only for an empty fleet (the lightest worker never clears
+    /// the mean), but the guard keeps a selection from returning no worker.
+    fn ungated_candidates(
+        &self,
+        workers: &[Arc<dyn Worker>],
+        healthy_indices: &[usize],
+        avg_load: f64,
+    ) -> Vec<usize> {
+        let ungated: Vec<usize> = healthy_indices
             .iter()
             .copied()
             .filter(|&idx| !self.candidate_requires_spill(workers, idx, avg_load))
             .collect();
-        let candidates = if spill_candidates.is_empty() {
-            healthy_indices
+        if ungated.is_empty() {
+            healthy_indices.to_vec()
         } else {
-            &spill_candidates
-        };
-        self.select_expected_wait(workers, candidates, info)
+            ungated
+        }
     }
 
     /// Pick an effective-affinity score group while preserving temperature.
@@ -1526,8 +1542,10 @@ impl CacheAwarePolicy {
             return Some(idx);
         }
 
-        // No cache overlap — expected-wait fallback over the healthy fleet.
-        let selected = self.select_expected_wait(workers, healthy_indices, info)?;
+        // No cache overlap — expected-wait fallback over the healthy fleet,
+        // bounded by the count-pressure gate like a tree miss.
+        let candidates = self.ungated_candidates(workers, healthy_indices, avg_load);
+        let selected = self.select_expected_wait(workers, &candidates, info)?;
         debug!(
             worker = workers[selected].url(),
             model_id, "Event-driven routing: no overlap, expected-wait fallback"
@@ -1821,10 +1839,13 @@ impl CacheAwarePolicy {
             return Some(selected);
         }
 
+        // No live holder at any boundary: bounded by the count-pressure gate
+        // like a tree miss.
+        let candidates = self.ungated_candidates(workers, healthy_indices, avg_load);
         self.hash_expected_wait(
             workers,
             info,
-            healthy_indices,
+            &candidates,
             model_id,
             "expected_wait_fallback",
             tokens,
@@ -1833,21 +1854,21 @@ impl CacheAwarePolicy {
         )
     }
 
-    /// Expected-wait dispatch for hash-path fallback branches; records only
-    /// the final worker when boundaries apply.
+    /// Expected-wait dispatch over `candidates` for hash-path fallback
+    /// branches; records only the final worker when boundaries apply.
     #[expect(clippy::too_many_arguments, reason = "hot-path plumbing, not state")]
     fn hash_expected_wait(
         &self,
         workers: &[Arc<dyn Worker>],
         info: &SelectWorkerInfo,
-        healthy_indices: &[usize],
+        candidates: &[usize],
         model_id: &str,
         branch: &'static str,
         tokens: &[u32],
         applicable: &[usize],
         now: Instant,
     ) -> Option<usize> {
-        let idx = self.select_expected_wait(workers, healthy_indices, info)?;
+        let idx = self.select_expected_wait(workers, candidates, info)?;
         if !applicable.is_empty() {
             self.record_placement(
                 model_id,
@@ -2434,7 +2455,8 @@ mod tests {
     }
 
     /// Single-DP snapshot used to make live request count and LeastLoad's
-    /// expected-wait score disagree deterministically.
+    /// expected-wait score disagree deterministically. A request is running,
+    /// so the rate gauge counts as live.
     fn expected_wait_load(
         queued_tokens: i32,
         token_usage: f64,
@@ -2442,6 +2464,7 @@ mod tests {
     ) -> WorkerLoadResponse {
         WorkerLoadResponse {
             loads: vec![SchedulerLoadSnapshot {
+                num_running_reqs: 1,
                 num_waiting_uncached_tokens: queued_tokens,
                 token_usage,
                 gen_throughput,
@@ -3422,6 +3445,148 @@ mod tests {
         assert_eq!(policy.select_worker(&workers, &info).unwrap(), 1);
     }
 
+    /// Two busy, empty-queued workers whose live rates differ so much that
+    /// expected wait alone would send every miss to w1.
+    fn skewed_rate_loads(workers: &[Arc<dyn Worker>]) -> HashMap<String, WorkerLoadResponse> {
+        HashMap::from([
+            (
+                workers[0].url().to_string(),
+                expected_wait_load(0, 0.1, 10_000.0),
+            ),
+            (
+                workers[1].url().to_string(),
+                expected_wait_load(0, 0.1, 100.0),
+            ),
+        ])
+    }
+
+    /// Gate at the mean (no margins), so an above-mean worker sits out.
+    fn gate_at_mean(config: CacheAwareConfig) -> CacheAwareConfig {
+        CacheAwareConfig {
+            balance_abs_threshold: 0,
+            balance_rel_threshold: 1.0,
+            ..config
+        }
+    }
+
+    /// A request at or below `cache_threshold` has no holder to spill from,
+    /// and expected wait alone keeps no count bound: its queue and rate
+    /// signals are a poll old, so a burst of misses followed one worker
+    /// while the other had free slots. Misses route among the workers under
+    /// the same count-pressure gate a hit spills at.
+    #[test]
+    fn miss_burst_is_bounded_by_the_count_pressure_gate() {
+        let policy = CacheAwarePolicy::with_config(gate_at_mean(test_config()));
+        let workers = make_workers(&["http://w1:8000", "http://w2:8000"]);
+        policy.init_workers(&workers);
+        policy.update_loads(&skewed_rate_loads(&workers));
+
+        let mut picks = [0usize; 2];
+        for i in 0..8 {
+            let text = format!("{i} novel prompt with nothing in common");
+            let idx = policy
+                .select_worker(
+                    &workers,
+                    &SelectWorkerInfo {
+                        request_text: Some(&text),
+                        ..Default::default()
+                    },
+                )
+                .unwrap();
+            // The router counts the dispatch in flight; the gate reads it.
+            workers[idx].increment_load();
+            picks[idx] += 1;
+        }
+        assert_eq!(picks, [4, 4]);
+    }
+
+    #[test]
+    fn event_miss_burst_is_bounded_by_the_count_pressure_gate() {
+        let policy = CacheAwarePolicy::with_config(gate_at_mean(test_config()));
+        let workers = make_workers(&["http://w1:8000", "http://w2:8000"]);
+        policy.init_workers(&workers);
+        policy.update_loads(&skewed_rate_loads(&workers));
+        let monitor = Arc::new(KvEventMonitor::new(Some(4)));
+        let indexer = setup_indexer_with_blocks("http://w1:8000", &[&[1, 2, 3, 4]], 4);
+        monitor.indexers.insert("unknown".to_string(), indexer);
+        policy.set_kv_event_monitor(Some(monitor));
+
+        let mut picks = [0usize; 2];
+        for i in 0..8u32 {
+            let novel = [100 + i, 200, 300, 400];
+            let idx = policy
+                .select_worker(
+                    &workers,
+                    &SelectWorkerInfo {
+                        tokens: Some(&novel),
+                        ..Default::default()
+                    },
+                )
+                .unwrap();
+            workers[idx].increment_load();
+            picks[idx] += 1;
+        }
+        assert_eq!(picks, [4, 4]);
+    }
+
+    #[test]
+    fn hash_miss_burst_is_bounded_by_the_count_pressure_gate() {
+        let policy = CacheAwarePolicy::with_config(gate_at_mean(hash_config(&[4])));
+        let workers = make_workers(&["http://w1:8000", "http://w2:8000"]);
+        policy.init_workers(&workers);
+        policy.update_loads(&skewed_rate_loads(&workers));
+
+        let mut picks = [0usize; 2];
+        for i in 0..8u32 {
+            let head = [i, i, i, i, 9];
+            let idx = route_tokens(&policy, &workers, &head);
+            workers[idx].increment_load();
+            picks[idx] += 1;
+        }
+        assert_eq!(picks, [4, 4]);
+    }
+
+    /// Two idle SGLang workers, one reporting the ~0.36 tok/s residual of
+    /// its last one-token health generation, the other nothing. Scored
+    /// against the residual, one dispatch read as ~2800 s of wait and the
+    /// whole burst piled onto the other worker. An idle rank has no live
+    /// rate, so both score at the default throughput and the misses
+    /// water-fill without the gate's help.
+    #[test]
+    fn idle_rate_residual_does_not_herd_misses() {
+        let policy = CacheAwarePolicy::with_config(test_config());
+        let workers = make_workers(&["http://w1:8000", "http://w2:8000"]);
+        policy.init_workers(&workers);
+        let idle = |gen_throughput| WorkerLoadResponse {
+            loads: vec![SchedulerLoadSnapshot {
+                num_running_reqs: 0,
+                gen_throughput,
+                ..Default::default()
+            }],
+            ..Default::default()
+        };
+        policy.update_loads(&HashMap::from([
+            (workers[0].url().to_string(), idle(0.36)),
+            (workers[1].url().to_string(), idle(0.0)),
+        ]));
+
+        let mut picks = [0usize; 2];
+        for i in 0..8 {
+            let text = format!("{i} novel prompt with nothing in common");
+            let idx = policy
+                .select_worker(
+                    &workers,
+                    &SelectWorkerInfo {
+                        request_text: Some(&text),
+                        ..Default::default()
+                    },
+                )
+                .unwrap();
+            picks[idx] += 1;
+        }
+        assert_eq!(picks, [4, 4]);
+    }
+
     #[test]
     fn kv_pressure_still_forces_expected_wait_fallback() {
         let policy = CacheAwarePolicy::with_config(CacheAwareConfig {
diff --git a/model_gateway/src/policies/least_load.rs b/model_gateway/src/policies/least_load.rs
index c1f37c7a9e..b6a9f8b0ea 100644
--- a/model_gateway/src/policies/least_load.rs
+++ b/model_gateway/src/policies/least_load.rs
@@ -51,9 +51,11 @@ struct SincePollDispatch {
 ///   `waiting_reqs · p̄`, keeping the queue visible in time units rather than
 ///   scoring a backlogged worker as idle. Only a backend reporting neither
 ///   scores `queued_tokens = 0`;
-/// - zero/absent throughput (backend reports no generation rate): falls back to
-///   the configured `default_throughput`, so the work term stays in seconds and
-///   the KV barrier stays relevant;
+/// - zero/absent throughput (backend reports no generation rate, or reports
+///   one for a rank with nothing running, which is a stale gauge rather than a
+///   rate; see [`live_throughput`]): falls back to the configured
+///   `default_throughput`, so the work term stays in seconds and the KV
+///   barrier stays relevant;
 /// - a worker with no fresh snapshot while peers report: its live in-flight is
 ///   converted to a drain-time estimate (`load · p̄ / fleet_nominal_throughput`)
 ///   so it is comparable to reporting workers, not scored on a raw count;
@@ -118,6 +120,25 @@ struct ScoreInputs<'a> {
     peer_baseline: f64,
 }
 
+/// Generation rate a report can be scored against: the sum over ranks with
+/// requests running.
+///
+/// An engine never clears `gen_throughput`: it is the rate of the last decode
+/// interval, so an idle rank reports a residual (SGLang: the ~0.3 tok/s of
+/// its last one-token health generation) or nothing at all. A dispatch
+/// divided by a residual reads as thousands of seconds, so a burst followed
+/// whichever worker happened to report the larger stale number until the
+/// next poll. A rank with nothing running contributes no rate; a worker with
+/// none scores at the fallback throughput, like one that never reports a
+/// rate.
+fn live_throughput(load: &WorkerLoadResponse) -> f64 {
+    load.loads
+        .iter()
+        .filter(|rank| rank.num_running_reqs > 0)
+        .map(|rank| rank.gen_throughput)
+        .sum()
+}
+
 impl LeastLoadPolicy {
     pub fn new() -> Self {
         Self::with_params(
@@ -214,9 +235,9 @@ impl LeastLoadPolicy {
             Some(load) => {
                 let inflight_tokens = inflight.get(url).copied().unwrap_or_default().tokens;
                 let queued_tokens = self.queued_tokens(load);
-                let live_throughput = load.total_gen_throughput();
-                let throughput = if live_throughput > 0.0 {
-                    live_throughput
+                let live = live_throughput(load);
+                let throughput = if live > 0.0 {
+                    live
                 } else {
                     self.default_throughput
                 };
@@ -355,7 +376,7 @@ impl LeastLoadPolicy {
         let (tp_sum, tp_count) = candidates
             .iter()
             .filter_map(|&i| Self::fresh_load(loads, complete_snapshot, workers[i].url()))
-            .map(|l| l.total_gen_throughput())
+            .map(live_throughput)
             .filter(|t| *t > 0.0)
             .fold((0.0, 0u32), |(s, n), t| (s + t, n + 1));
         let nominal_throughput = if tp_count > 0 {
@@ -551,7 +572,8 @@ mod tests {
         }
     }
 
-    /// One DP rank with the given queued tokens, KV utilization, and throughput.
+    /// One DP rank with the given queued tokens, KV utilization, and live
+    /// throughput (a request is running, so the rate gauge counts).
     fn make_load(
         num_waiting_uncached_tokens: i32,
         token_usage: f64,
@@ -562,7 +584,7 @@ mod tests {
             dp_rank_count: 1,
             loads: vec![SchedulerLoadSnapshot {
                 dp_rank: 0,
-                num_running_reqs: 0,
+                num_running_reqs: 1,
                 num_waiting_reqs: 0,
                 num_waiting_uncached_tokens,
                 num_total_reqs: 0,
@@ -956,6 +978,52 @@ mod tests {
         );
     }
 
+    #[test]
+    fn idle_rank_rate_is_not_a_live_throughput() {
+        // Only a rank with requests running has a live rate; an idle rank's
+        // gauge is whatever its last decode interval left behind.
+        let mut load = make_load(0, 0.1, 0.36);
+        load.loads[0].num_running_reqs = 0;
+        assert_eq!(live_throughput(&load), 0.0);
+
+        load.loads.push(SchedulerLoadSnapshot {
+            dp_rank: 1,
+            num_running_reqs: 4,
+            gen_throughput: 800.0,
+            ..Default::default()
+        });
+        assert_eq!(live_throughput(&load), 800.0);
+    }
+
+    #[test]
+    fn stale_idle_rate_does_not_herd_a_burst() {
+        // Two idle workers: a reports the ~0.36 tok/s residual of its last
+        // one-token health generation, b reports nothing. Scored against the
+        // residual, a's first dispatch would read as 1024 / 0.36 ≈ 2800 s and
+        // the rest of the burst would follow b until the next poll. Neither
+        // rate is live, so both score at the default throughput and the
+        // burst water-fills.
+        let policy = LeastLoadPolicy::new();
+        let workers = vec![mk("http://a:8000"), mk("http://b:8000")];
+        let mut a = make_load(0, 0.1, 0.36);
+        a.loads[0].num_running_reqs = 0;
+        let mut b = make_load(0, 0.1, 0.0);
+        b.loads[0].num_running_reqs = 0;
+        policy.update_loads(&HashMap::from([
+            ("http://a:8000".to_string(), a),
+            ("http://b:8000".to_string(), b),
+        ]));
+
+        let mut picks = [0usize; 2];
+        for _ in 0..8 {
+            let idx = policy
+                .select_worker(&workers, &SelectWorkerInfo::default())
+                .unwrap();
+            picks[idx] += 1;
+        }
+        assert_eq!(picks, [4, 4]);
+    }
+
     #[test]
     fn zero_throughput_falls_back_to_default() {
         // A backend that reports no gen_throughput (0); the score must still
diff --git a/model_gateway/src/policies/mod.rs b/model_gateway/src/policies/mod.rs
index 94c034af28..3e93b49648 100644
--- a/model_gateway/src/policies/mod.rs
+++ b/model_gateway/src/policies/mod.rs
@@ -136,7 +136,9 @@ pub struct CacheAwareConfig {
     /// mean by this many requests AND by `balance_rel_threshold`. Requiring
     /// both keeps the gate quiet on steady-state variance at low means
     /// (absolute) without going blind to a deep queue at high means
-    /// (relative).
+    /// (relative). A request with no cache match routes among the workers
+    /// under the same gate, so a burst of misses is bounded by the same
+    /// margins.
     pub balance_abs_threshold: usize,
     /// Relative load margin (multiple of the healthy-fleet mean) for the
     /// per-request candidate gate; fires only together with
diff --git a/model_gateway/src/policies/power_of_two.rs b/model_gateway/src/policies/power_of_two.rs
index 8635432729..757ee6954b 100644
--- a/model_gateway/src/policies/power_of_two.rs
+++ b/model_gateway/src/policies/power_of_two.rs
@@ -92,7 +92,8 @@ mod tests {
         }
     }
 
-    /// One DP rank with the given queued tokens, KV utilization, and throughput.
+    /// One DP rank with the given queued tokens, KV utilization, and live
+    /// throughput (a request is running, so the rate gauge counts).
     fn make_load(
         num_waiting_uncached_tokens: i32,
         token_usage: f64,
@@ -103,6 +104,7 @@ mod tests {
             dp_rank_count: 1,
             loads: vec![SchedulerLoadSnapshot {
                 dp_rank: 0,
+                num_running_reqs: 1,
                 num_waiting_uncached_tokens,
                 token_usage,
                 gen_throughput,

From 047612b2af69acb5422842e618fa80ae8cde4b20 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Tue, 6 Oct 2026 13:55:38 +0000
Subject: [PATCH 2/4] fix(cache_aware): gate hash-mode head misses and score
 idle workers at the fleet rate

Hash mode checked KV pressure only after its untokenized and head-only
early returns, so those two misses (every HTTP-router request, in hash
mode) kept the ungated expected-wait selection. Check KV pressure first,
as the tree path does, then route both through the count-pressure gate
like any other miss.

A worker with no live rate was scored at the flat default throughput,
which beside busy peers made it look slower per dispatched token than
they are: at 8000 tok/s live against a 2000 default, the busy worker
took four of every five dispatches. Score it at the fleet's nominal
rate, the mean live rate among the candidates, which is the default
only when no candidate has one.

Refs: #2804
---
 model_gateway/src/policies/cache_aware.rs | 73 +++++++++++++++--------
 model_gateway/src/policies/least_load.rs  | 52 ++++++++++++----
 2 files changed, 88 insertions(+), 37 deletions(-)

diff --git a/model_gateway/src/policies/cache_aware.rs b/model_gateway/src/policies/cache_aware.rs
index e2e1d831b3..d706377e9e 100644
--- a/model_gateway/src/policies/cache_aware.rs
+++ b/model_gateway/src/policies/cache_aware.rs
@@ -1381,9 +1381,10 @@ impl CacheAwarePolicy {
         self.select_expected_wait(workers, &candidates, info)
     }
 
-    /// The healthy workers under the count-pressure gate. Every worker
-    /// trips it only for an empty fleet (the lightest worker never clears
-    /// the mean), but the guard keeps a selection from returning no worker.
+    /// The healthy workers under the count-pressure gate. The lightest worker
+    /// never clears the snapshot mean, but `avg_load` and the per-worker load
+    /// re-read can disagree under concurrent dispatch, so the guard keeps a
+    /// selection from returning no worker.
     fn ungated_candidates(
         &self,
         workers: &[Arc<dyn Worker>],
@@ -1755,50 +1756,47 @@ impl CacheAwarePolicy {
         // Hash mode keys on token ids; untokenized requests stay load-balanced.
         // Unpartitioned heads are stripped of marker material like the tree
         // keys, so the placement key spaces stay disjoint by construction too.
-        let tokens = match info.cache_namespace {
+        let tokens: &[u32] = match info.cache_namespace {
             Some(_) => info.tokens,
             None => info.tokens.map(CacheNamespace::unpartitioned_tokens),
-        };
-        let Some(tokens) = tokens.filter(|t| !t.is_empty()) else {
-            return self.hash_expected_wait(
-                workers,
-                info,
-                healthy_indices,
-                model_id,
-                "expected_wait_fallback",
-                &[],
-                &[],
-                now,
-            );
-        };
-
+        }
+        .unwrap_or(&[]);
         let applicable_end = self
             .config
             .cache_boundaries
             .partition_point(|&p| p <= tokens.len());
         let applicable = &self.config.cache_boundaries[..applicable_end];
 
-        // Head-only traffic must stay load-balanced.
-        if applicable.is_empty() {
+        // Backend KV pressure sheds fleet-wide ahead of any count gate, as the
+        // tree path does for every request it routes.
+        if self.is_kv_imbalanced(workers, healthy_indices) {
             return self.hash_expected_wait(
                 workers,
                 info,
                 healthy_indices,
                 model_id,
-                "short_request",
+                "kv_pressure_expected_wait",
                 tokens,
                 applicable,
                 now,
             );
         }
 
-        if self.is_kv_imbalanced(workers, healthy_indices) {
+        // Untokenized or head-only traffic has no holder to find: it stays
+        // load-balanced, bounded by the count-pressure gate like a tree miss.
+        if applicable.is_empty() {
+            let candidates = self.ungated_candidates(workers, healthy_indices, avg_load);
+            let branch = if tokens.is_empty() {
+                "expected_wait_fallback"
+            } else {
+                "short_request"
+            };
             return self.hash_expected_wait(
                 workers,
                 info,
-                healthy_indices,
+                &candidates,
                 model_id,
-                "kv_pressure_expected_wait",
+                branch,
                 tokens,
                 applicable,
                 now,
@@ -3546,6 +3544,33 @@ mod tests {
         assert_eq!(picks, [4, 4]);
     }
 
+    /// Hash mode keys on token ids, so every HTTP-router request is a miss
+    /// there; the same bound applies.
+    #[test]
+    fn hash_untokenized_burst_is_bounded_by_the_count_pressure_gate() {
+        let policy = CacheAwarePolicy::with_config(gate_at_mean(hash_config(&[4])));
+        let workers = make_workers(&["http://w1:8000", "http://w2:8000"]);
+        policy.init_workers(&workers);
+        policy.update_loads(&skewed_rate_loads(&workers));
+
+        let mut picks = [0usize; 2];
+        for i in 0..8 {
+            let text = format!("{i} novel prompt with nothing in common");
+            let idx = policy
+                .select_worker(
+                    &workers,
+                    &SelectWorkerInfo {
+                        request_text: Some(&text),
+                        ..Default::default()
+                    },
+                )
+                .unwrap();
+            workers[idx].increment_load();
+            picks[idx] += 1;
+        }
+        assert_eq!(picks, [4, 4]);
+    }
+
     /// Two idle SGLang workers, one reporting the ~0.36 tok/s residual of
     /// its last one-token health generation, the other nothing. Scored
     /// against the residual, one dispatch read as ~2800 s of wait and the
diff --git a/model_gateway/src/policies/least_load.rs b/model_gateway/src/policies/least_load.rs
index b6a9f8b0ea..e84d40ca06 100644
--- a/model_gateway/src/policies/least_load.rs
+++ b/model_gateway/src/policies/least_load.rs
@@ -53,9 +53,10 @@ struct SincePollDispatch {
 ///   scores `queued_tokens = 0`;
 /// - zero/absent throughput (backend reports no generation rate, or reports
 ///   one for a rank with nothing running, which is a stale gauge rather than a
-///   rate; see [`live_throughput`]): falls back to the configured
-///   `default_throughput`, so the work term stays in seconds and the KV
-///   barrier stays relevant;
+///   rate; see [`live_throughput`]): scored at the fleet's nominal rate, the
+///   mean live rate among the candidates, or the configured
+///   `default_throughput` when none has one, so the work term stays in
+///   seconds and the KV barrier stays relevant;
 /// - a worker with no fresh snapshot while peers report: its live in-flight is
 ///   converted to a drain-time estimate (`load · p̄ / fleet_nominal_throughput`)
 ///   so it is comparable to reporting workers, not scored on a raw count;
@@ -73,8 +74,8 @@ struct SincePollDispatch {
 /// - `kv_pressure_weight` (λ_t, default `0.15` s) — weight of the KV-pressure
 ///   barrier. Raise it to steer harder away from near-full KV; lower it to
 ///   weight raw drain time more.
-/// - `default_throughput` (default `2000` tok/s) — drain rate used when a
-///   backend reports no live `gen_throughput`. Set it to the fleet's measured
+/// - `default_throughput` (default `2000` tok/s) — drain rate used when no
+///   candidate reports a live `gen_throughput`. Set it to the fleet's measured
 ///   per-replica generation rate; it co-tunes with `kv_pressure_weight`.
 /// - `mean_prefill_tokens` (p̄, default `1024`) — per-request token estimate for
 ///   the in-flight term when the request's token count is unknown at routing
@@ -100,8 +101,8 @@ pub struct LeastLoadPolicy {
     /// Mean prefill length (tokens) for estimating in-flight token-work when a
     /// request's token count is unknown at routing time.
     mean_prefill_tokens: u32,
-    /// Fallback throughput (tokens/s) for the `/throughput` term when a backend
-    /// reports no live `gen_throughput`.
+    /// Fallback throughput (tokens/s) for the `/throughput` term when no
+    /// candidate reports a live `gen_throughput`.
     default_throughput: f64,
     /// Per-worker waiting-queue cap; `0` disables the veto.
     max_waiting_requests: u32,
@@ -129,7 +130,7 @@ struct ScoreInputs<'a> {
 /// divided by a residual reads as thousands of seconds, so a burst followed
 /// whichever worker happened to report the larger stale number until the
 /// next poll. A rank with nothing running contributes no rate; a worker with
-/// none scores at the fallback throughput, like one that never reports a
+/// none is scored at the fleet's nominal rate, like one that never reports a
 /// rate.
 fn live_throughput(load: &WorkerLoadResponse) -> f64 {
     load.loads
@@ -235,12 +236,12 @@ impl LeastLoadPolicy {
             Some(load) => {
                 let inflight_tokens = inflight.get(url).copied().unwrap_or_default().tokens;
                 let queued_tokens = self.queued_tokens(load);
+                // No live rate (none reported, or only idle ranks): the
+                // fleet's nominal rate, so an idle worker beside busy peers
+                // reads as fast as they are, not slower. With no live rate
+                // anywhere that is the configured default.
                 let live = live_throughput(load);
-                let throughput = if live > 0.0 {
-                    live
-                } else {
-                    self.default_throughput
-                };
+                let throughput = if live > 0.0 { live } else { nominal_throughput };
                 ExpectedWait::new(
                     queued_tokens,
                     throughput,
@@ -1024,6 +1025,31 @@ mod tests {
         assert_eq!(picks, [4, 4]);
     }
 
+    #[test]
+    fn idle_worker_is_scored_at_its_busy_peers_rate() {
+        // a is busy at 8000 tok/s; b is idle, its gauge a stale residual. At a
+        // flat 2000 tok/s a dispatch to b would read as four times one to a
+        // and a would take four of every five; at its peers' nominal rate the
+        // two score alike and the burst water-fills.
+        let policy = LeastLoadPolicy::new();
+        let workers = vec![mk("http://a:8000"), mk("http://b:8000")];
+        let mut b = make_load(0, 0.1, 0.36);
+        b.loads[0].num_running_reqs = 0;
+        policy.update_loads(&HashMap::from([
+            ("http://a:8000".to_string(), make_load(0, 0.1, 8000.0)),
+            ("http://b:8000".to_string(), b),
+        ]));
+
+        let mut picks = [0usize; 2];
+        for _ in 0..8 {
+            let idx = policy
+                .select_worker(&workers, &SelectWorkerInfo::default())
+                .unwrap();
+            picks[idx] += 1;
+        }
+        assert_eq!(picks, [4, 4]);
+    }
+
     #[test]
     fn zero_throughput_falls_back_to_default() {
         // A backend that reports no gen_throughput (0); the score must still

From 618040d910899950fc0999a8361d9b8dd8254143 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Tue, 6 Oct 2026 13:59:40 +0000
Subject: [PATCH 3/4] docs(least_load): say when the default throughput applies

Since the scorer falls back to the fleet's nominal rate, the configured
default only applies when no candidate reports a live rate; the CLI,
config and Python help strings still said "when a backend reports none".

Refs: #2804
---
 bindings/python/src/smg/router_args.py | 4 ++--
 model_gateway/src/config/types.rs      | 2 +-
 model_gateway/src/main.rs              | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/bindings/python/src/smg/router_args.py b/bindings/python/src/smg/router_args.py
index edf0a6537d..169f250b2d 100644
--- a/bindings/python/src/smg/router_args.py
+++ b/bindings/python/src/smg/router_args.py
@@ -530,8 +530,8 @@ def add_cli_args(
             type=float,
             default=RouterArgs.least_load_default_throughput,
             help=(
-                "Fallback generation throughput (tokens/s) for least_load when a"
-                " backend reports no live throughput"
+                "Fallback generation throughput (tokens/s) for least_load when no"
+                " candidate reports a live throughput"
             ),
         )
         routing_group.add_argument(
diff --git a/model_gateway/src/config/types.rs b/model_gateway/src/config/types.rs
index 238a9c4980..cbc341d2d9 100755
--- a/model_gateway/src/config/types.rs
+++ b/model_gateway/src/config/types.rs
@@ -799,7 +799,7 @@ pub enum PolicyConfig {
         #[serde(default = "default_least_load_mean_prefill")]
         mean_prefill_tokens: u32,
         /// Fallback generation throughput (tokens/s) for the expected-wait term
-        /// when a backend reports no live `gen_throughput`. Set to the fleet's
+        /// when no candidate reports a live `gen_throughput`. Set to the fleet's
         /// per-replica generation rate; co-tunes with `kv_pressure_weight`.
         #[serde(default = "default_least_load_throughput")]
         default_throughput: f64,
diff --git a/model_gateway/src/main.rs b/model_gateway/src/main.rs
index e4b9d8afd8..d75a970976 100644
--- a/model_gateway/src/main.rs
+++ b/model_gateway/src/main.rs
@@ -413,8 +413,8 @@ struct CliArgs {
     #[arg(long, default_value_t = 0.15, help_heading = "Routing Policy")]
     least_load_kv_pressure_weight: f64,
 
-    /// Fallback generation throughput (tokens/s) for least_load when a backend
-    /// reports no live throughput
+    /// Fallback generation throughput (tokens/s) for least_load when no
+    /// candidate reports a live throughput
     #[arg(long, default_value_t = 2000.0, help_heading = "Routing Policy")]
     least_load_default_throughput: f64,
 

From 8c70ec2e5adbdec2ce71662acd338f668f7ccd37 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Tue, 6 Oct 2026 14:20:31 +0000
Subject: [PATCH 4/4] docs(cache_aware): scope the miss-burst bound to the
 KV-balanced paths

The count-pressure gate bounds a burst of misses on the tree, event and
hash paths, but the fleet-wide KV-pressure fallback that runs ahead of
them selects from every healthy worker; say so where the bound is
described.

Refs: #2804
---
 model_gateway/src/policies/cache_aware.rs | 8 ++++----
 model_gateway/src/policies/mod.rs         | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/model_gateway/src/policies/cache_aware.rs b/model_gateway/src/policies/cache_aware.rs
index d706377e9e..2c2a5460fd 100644
--- a/model_gateway/src/policies/cache_aware.rs
+++ b/model_gateway/src/policies/cache_aware.rs
@@ -1333,10 +1333,10 @@ impl CacheAwarePolicy {
     /// hot prefixes replicate instead of queueing behind one engine. Both
     /// margins must clear so the gate neither fires on steady-state variance
     /// (relative alone would, at low means) nor stays blind to a deep queue
-    /// (absolute alone would, at high means). A request with no holder is
-    /// routed among the workers under the same gate
-    /// ([`Self::ungated_candidates`]), so a burst of misses is bounded by
-    /// the margins a hit spills at.
+    /// (absolute alone would, at high means). Outside the fleet-wide
+    /// KV-pressure fallback, a request with no holder is routed among the
+    /// workers under the same gate ([`Self::ungated_candidates`]), so a
+    /// burst of misses is bounded by the margins a hit spills at.
     fn candidate_requires_spill(
         &self,
         workers: &[Arc<dyn Worker>],
diff --git a/model_gateway/src/policies/mod.rs b/model_gateway/src/policies/mod.rs
index 3e93b49648..607cd1e274 100644
--- a/model_gateway/src/policies/mod.rs
+++ b/model_gateway/src/policies/mod.rs
@@ -136,9 +136,9 @@ pub struct CacheAwareConfig {
     /// mean by this many requests AND by `balance_rel_threshold`. Requiring
     /// both keeps the gate quiet on steady-state variance at low means
     /// (absolute) without going blind to a deep queue at high means
-    /// (relative). A request with no cache match routes among the workers
-    /// under the same gate, so a burst of misses is bounded by the same
-    /// margins.
+    /// (relative). Outside the fleet-wide KV-pressure fallback, a request
+    /// with no cache match routes among the workers under the same gate, so
+    /// a burst of misses is bounded by the same margins.
     pub balance_abs_threshold: usize,
     /// Relative load margin (multiple of the healthy-fleet mean) for the
     /// per-request candidate gate; fires only together with