sgl-project · chengcuiping · Aug 6, 2026 · Aug 6, 2026 · Aug 7, 2026 · Aug 10, 2026
@@ -27,7 +27,7 @@ High-performance model routing control and data plane for large-scale LLM deploy
 - Advanced load balancing with cache-aware request reuse, load-aware (power-of-two) selection, and per-model policy overrides.
 
 ## Feature Highlights
-- Multiple load balancing strategies (`random`, `round_robin`, `cache_aware`, `power_of_two`, `bucket`) with DP-aware scheduling.
+- Multiple load balancing strategies (`random`, `round_robin`, `cache_aware`, `power_of_two`, `bucket`, `consistent_hashing`, `bounded_consistent_hashing`) with DP-aware scheduling.
 - Multi-model HTTP serving and inference gateway routing with model-specific policies.
 - Prefill/decode disaggregation, including bootstrap port handling and cache-aware merging.
 - gRPC routing with fully Rust tokenizer loading, reasoning parser selection, and tool parser integration for OpenAI-compatible endpoints—supporting streaming and non-streaming modes across DeepSeek, Llama, Kimi K2, Qwen, GPT-OSS, Mistral, Step-3, GLM4, GLM4.7 and other reasoning-capable models.
@@ -738,6 +738,11 @@ Router flags map to these values:
 - `round_robin`: sequential rotation with atomic counters.
 - `cache_aware`: maintains a prefix tree of prompts to route repeat traffic and evens load with configurable thresholds (`--cache-threshold`, `--balance-abs-threshold`, `--balance-rel-threshold`, `--eviction-interval`, `--max-tree-size`).
 - `power_of_two`: chooses the lighter worker among two random candidates; integrates with `LoadMonitor`.
+- `consistent_hashing`: keeps an explicit `X-SMG-Routing-Key` on its preferred healthy worker and does not consider load.
+- `bounded_consistent_hashing`: an opt-in variant of consistent hashing. With an explicit `X-SMG-Routing-Key`, it spills only when both `preferred_load - min_healthy_load > min_load_gap` and `preferred_load > mean_healthy_load * max_load_skew`; `min_load_gap` is measured in active requests. It then walks the ring clockwise to the first healthy worker within the relative bound, retaining the preferred worker if no candidate qualifies.
+  `X-SMG-Target-Worker` and implicit keys from `Authorization`, `X-Forwarded-For`, or `Cookie` remain strict. The active-load signal is best-effort and local to each gateway process, and this soft-affinity policy must not be used when worker-local session state requires strict affinity.
+  Configure it with `--policy bounded_consistent_hashing --max-load-skew 1.5 --min-load-gap <active-requests>`.
+  The default `min_load_gap` is 2 as a conservative middle setting for this opt-in policy, not as an empirically optimal value. Operators should tune it for their worker count, concurrent fan-out, request length, and cache-reuse trade-off. Existing `consistent_hashing` behavior is unchanged.
   Per-model overrides are available in PD mode (`--prefill-policy`, `--decode-policy`) and IGW mode via the worker registry.
 
 ## Observability

@@ -15,6 +15,7 @@ pub enum PolicyType {
     Bucket,
     Manual,
     ConsistentHashing,
+    BoundedConsistentHashing,
     PrefixHash,
 }
 
@@ -355,6 +356,8 @@ struct Router {
     port: u16,
     worker_urls: Vec<String>,
     policy: PolicyType,
+    max_load_skew: f64,
+    min_load_gap: usize,
     worker_startup_timeout_secs: u64,
     worker_startup_check_interval: u64,
     cache_threshold: f32,
@@ -497,6 +500,12 @@ impl Router {
                     },
                 },
                 PolicyType::ConsistentHashing => ConfigPolicyConfig::ConsistentHashing,
+                PolicyType::BoundedConsistentHashing => {
+                    ConfigPolicyConfig::BoundedConsistentHashing {
+                        max_load_skew: self.max_load_skew,
+                        min_load_gap: self.min_load_gap,
+                    }
+                }
                 PolicyType::PrefixHash => ConfigPolicyConfig::PrefixHash {
                     prefix_token_count: 256,
                     load_factor: 1.25,
@@ -761,6 +770,8 @@ impl Router {
         pool_max_idle_per_host = 500,
         tcp_keepalive_secs = 30,
         enable_wasm = false,
+        max_load_skew = 1.5,
+        min_load_gap = 2,
     ))]
     #[allow(clippy::too_many_arguments)]
     fn new(
@@ -853,6 +864,8 @@ impl Router {
         pool_max_idle_per_host: usize,
         tcp_keepalive_secs: u64,
         enable_wasm: bool,
+        max_load_skew: f64,
+        min_load_gap: usize,
     ) -> PyResult<Self> {
         let mut all_urls = worker_urls.clone();
 
@@ -873,6 +886,8 @@ impl Router {
             port,
             worker_urls,
             policy,
+            max_load_skew,
+            min_load_gap,
             worker_startup_timeout_secs,
             worker_startup_check_interval,
             cache_threshold,

@@ -31,6 +31,7 @@ def policy_from_str(policy_str: Optional[str]) -> PolicyType:
         "bucket": PolicyType.Bucket,
         "manual": PolicyType.Manual,
         "consistent_hashing": PolicyType.ConsistentHashing,
+        "bounded_consistent_hashing": PolicyType.BoundedConsistentHashing,
         "prefix_hash": PolicyType.PrefixHash,
     }
     return policy_map[policy_str]
@@ -156,6 +157,9 @@ class Router:
             - PolicyType.RoundRobin: Distribute requests in round-robin fashion
             - PolicyType.CacheAware: Distribute requests based on cache state and load balance
             - PolicyType.PowerOfTwo: Select best of two random workers based on load (PD mode only)
+            - PolicyType.BoundedConsistentHashing: Keep explicit routing-key affinity within a bounded worker load skew
+        max_load_skew: Active-load bound for bounded consistent hashing. Default: 1.5
+        min_load_gap: Minimum preferred-to-least-loaded worker gap, in active requests, before spillover is allowed.
         host: Host address to bind the router server. Supports IPv4, IPv6 (e.g., ::, ::1), or 0.0.0.0 for all interfaces. Default: '0.0.0.0'
         port: Port number to bind the router server. Default: 3001
         worker_startup_timeout_secs: Timeout in seconds for worker startup and registration. Large models can take significant time to load into GPU memory. Default: 1800 (30 minutes)

@@ -30,6 +30,7 @@ def get_available_tool_call_parsers() -> List[str]:
     "bucket",
     "manual",
     "consistent_hashing",
+    "bounded_consistent_hashing",
     "prefix_hash",
 )
 
@@ -59,6 +60,10 @@ class RouterArgs:
     cache_threshold: float = 0.3
     balance_abs_threshold: int = 64
     balance_rel_threshold: float = 1.5
+    max_load_skew: float = 1.5
+    min_load_gap: int = (
+        2  # Conservative configurable default for opt-in bounded routing
+    )
     eviction_interval_secs: int = 60
     max_tree_size: int = 2**26
     max_idle_secs: int = 4 * 3600
@@ -319,6 +324,18 @@ def add_cli_args(
             default=RouterArgs.balance_rel_threshold,
             help="Relative threshold for load difference. Balancing is triggered if `max_load > min_load * rel_threshold` and the absolute threshold is also met.",
         )
+        routing_group.add_argument(
+            f"--{prefix}max-load-skew",
+            type=float,
+            default=RouterArgs.max_load_skew,
+            help="Maximum preferred-worker load relative to the healthy-worker mean for bounded_consistent_hashing",
+        )
+        routing_group.add_argument(
+            f"--{prefix}min-load-gap",
+            type=int,
+            default=RouterArgs.min_load_gap,
+            help="Minimum active-request gap between the preferred and least-loaded healthy worker before bounded_consistent_hashing may spill",
+        )
         routing_group.add_argument(
             f"--{prefix}bucket-adjust-interval-secs",
             type=int,

@@ -443,6 +443,22 @@ def test_parse_basic_args(self):
         assert router_args.worker_urls == ["http://worker1:8000", "http://worker2:8000"]
         assert router_args.policy == "round_robin"
 
+    def test_parse_bounded_hashing_thresholds(self):
+        router_args = parse_router_args(
+            [
+                "--policy",
+                "bounded_consistent_hashing",
+                "--max-load-skew",
+                "1.75",
+                "--min-load-gap",
+                "4",
+            ]
+        )
+
+        assert router_args.policy == "bounded_consistent_hashing"
+        assert router_args.max_load_skew == 1.75
+        assert router_args.min_load_gap == 4
+
     def test_parse_pd_args(self):
         """Test parsing PD disaggregated mode arguments."""
         args = [

@@ -46,6 +46,7 @@ def test_policy_from_str_covers_all_variants(self):
             "bucket": PolicyType.Bucket,
             "manual": PolicyType.Manual,
             "consistent_hashing": PolicyType.ConsistentHashing,
+            "bounded_consistent_hashing": PolicyType.BoundedConsistentHashing,
             "prefix_hash": PolicyType.PrefixHash,
         }
         for s, expected in cases.items():
@@ -355,11 +356,14 @@ def test_all_policies_construct(self):
             "bucket",
             "manual",
             "consistent_hashing",
+            "bounded_consistent_hashing",
             "prefix_hash",
         ):
             args = RouterArgs(
                 worker_urls=["http://w1:8000"],
                 policy=policy,
+                max_load_skew=1.75,
+                min_load_gap=4,
                 pd_disaggregation=True,
                 prefill_urls=[("http://prefill1:8000", None)],
                 decode_urls=["http://decode1:8001"],
@@ -658,6 +662,7 @@ class TestPolicyChoiceListConsistency:
             "bucket",
             "manual",
             "consistent_hashing",
+            "bounded_consistent_hashing",
             "prefix_hash",
         ],
     )
@@ -677,6 +682,7 @@ def test_main_policy_accepts(self, policy):
             "bucket",
             "manual",
             "consistent_hashing",
+            "bounded_consistent_hashing",
             "prefix_hash",
         ],
     )

@@ -322,7 +322,13 @@ def test_pd_service_discovery_validation(self):
     def test_policy_validation(self):
         """Test policy configuration validation."""
         # Valid policies
-        valid_policies = ["random", "round_robin", "cache_aware", "power_of_two"]
+        valid_policies = [
+            "random",
+            "round_robin",
+            "cache_aware",
+            "power_of_two",
+            "bounded_consistent_hashing",
+        ]
 
         for policy in valid_policies:
             args = RouterArgs(policy=policy)
@@ -331,7 +337,13 @@ def test_policy_validation(self):
     def test_pd_policy_validation(self):
         """Test PD policy configuration validation."""
         # Valid PD policies
-        valid_policies = ["random", "round_robin", "cache_aware", "power_of_two"]
+        valid_policies = [
+            "random",
+            "round_robin",
+            "cache_aware",
+            "power_of_two",
+            "bounded_consistent_hashing",
+        ]
 
         for prefill_policy in valid_policies:
             for decode_policy in valid_policies:

@@ -108,6 +108,18 @@ impl RouterConfigBuilder {
         self
     }
 
+    pub fn bounded_consistent_hashing_policy(
+        mut self,
+        max_load_skew: f64,
+        min_load_gap: usize,
+    ) -> Self {
+        self.config.policy = PolicyConfig::BoundedConsistentHashing {
+            max_load_skew,
+            min_load_gap,
+        };
+        self
+    }
+
     pub fn cache_aware_policy(
         mut self,
         cache_threshold: f32,

@@ -305,6 +305,24 @@ pub enum PolicyConfig {
     #[serde(rename = "consistent_hashing")]
     ConsistentHashing,
 
+    /// Opt-in consistent hashing with bounded load skew for explicit routing keys.
+    ///
+    /// The preferred worker is selected from the consistent hash ring. If it is
+    /// above the configured load bound, the ring is walked clockwise to find the
+    /// first healthy worker within the bound.
+    #[serde(rename = "bounded_consistent_hashing")]
+    BoundedConsistentHashing {
+        /// Maximum worker load relative to the average healthy worker load.
+        /// Defaults to 1.5 (150% of average).
+        #[serde(default = "default_max_load_skew")]
+        max_load_skew: f64,
+
+        /// Minimum active-request gap between the preferred worker and the
+        /// least-loaded healthy worker before spillover is allowed.
+        #[serde(default = "default_min_load_gap")]
+        min_load_gap: usize,
+    },
+
     /// Prefix hash policy for KV cache-aware load balancing.
     /// A lightweight alternative to cache_aware radix tree.
     /// Routes requests based on prefix token hash for cache locality.
@@ -326,6 +344,15 @@ fn default_prefix_token_count() -> usize {
     256
 }
 
+fn default_max_load_skew() -> f64 {
+    1.5
+}
+
+// Conservative configurable default for this opt-in policy.
+fn default_min_load_gap() -> usize {
+    2
+}
+
 fn default_load_factor() -> f64 {
     1.25
 }
@@ -348,6 +375,7 @@ impl PolicyConfig {
             PolicyConfig::Bucket { .. } => "bucket",
             PolicyConfig::Manual { .. } => "manual",
             PolicyConfig::ConsistentHashing => "consistent_hashing",
+            PolicyConfig::BoundedConsistentHashing { .. } => "bounded_consistent_hashing",
             PolicyConfig::PrefixHash { .. } => "prefix_hash",
         }
     }
@@ -819,6 +847,12 @@ mod tests {
             load_check_interval_secs: 60,
         };
         assert_eq!(power_of_two.name(), "power_of_two");
+
+        let bounded = PolicyConfig::BoundedConsistentHashing {
+            max_load_skew: 1.5,
+            min_load_gap: 2,
+        };
+        assert_eq!(bounded.name(), "bounded_consistent_hashing");
     }
 
     #[test]
@@ -845,6 +879,28 @@ mod tests {
         let json = serde_json::to_string(&power_of_two).unwrap();
         assert!(json.contains("\"type\":\"power_of_two\""));
         assert!(json.contains("\"load_check_interval_secs\":60"));
+
+        let bounded = PolicyConfig::BoundedConsistentHashing {
+            max_load_skew: 1.75,
+            min_load_gap: 4,
+        };
+        let json = serde_json::to_string(&bounded).unwrap();
+        assert!(json.contains("\"type\":\"bounded_consistent_hashing\""));
+        assert!(json.contains("\"max_load_skew\":1.75"));
+        assert!(json.contains("\"min_load_gap\":4"));
+
+        let default_bounded: PolicyConfig =
+            serde_json::from_str(r#"{"type":"bounded_consistent_hashing"}"#).unwrap();
+        match default_bounded {
+            PolicyConfig::BoundedConsistentHashing {
+                max_load_skew,
+                min_load_gap,
+            } => {
+                assert!((max_load_skew - 1.5).abs() < f64::EPSILON);
+                assert_eq!(min_load_gap, 2);
+            }
+            _ => panic!("Expected bounded consistent hashing policy"),
+        }
     }
 
     #[test]

@@ -151,6 +151,18 @@ impl ConfigValidator {
             | PolicyConfig::RoundRobin
             | PolicyConfig::Manual { .. }
             | PolicyConfig::ConsistentHashing => {}
+            PolicyConfig::BoundedConsistentHashing {
+                max_load_skew,
+                min_load_gap: _,
+            } => {
+                if !max_load_skew.is_finite() || *max_load_skew < 1.0 {
+                    return Err(ConfigError::InvalidValue {
+                        field: "max_load_skew".to_string(),
+                        value: max_load_skew.to_string(),
+                        reason: "Must be a finite value >= 1.0".to_string(),
+                    });
+                }
+            }
             PolicyConfig::CacheAware {
                 cache_threshold,
                 balance_abs_threshold: _,
@@ -667,6 +679,31 @@ mod tests {
         assert!(ConfigValidator::validate(&config).is_ok());
     }
 
+    #[test]
+    fn test_validate_bounded_consistent_hashing_skew() {
+        assert!(
+            ConfigValidator::validate_policy(&PolicyConfig::BoundedConsistentHashing {
+                max_load_skew: 1.0,
+                min_load_gap: 2,
+            })
+            .is_ok()
+        );
+        assert!(
+            ConfigValidator::validate_policy(&PolicyConfig::BoundedConsistentHashing {
+                max_load_skew: 0.99,
+                min_load_gap: 2,
+            })
+            .is_err()
+        );
+        assert!(
+            ConfigValidator::validate_policy(&PolicyConfig::BoundedConsistentHashing {
+                max_load_skew: f64::NAN,
+                min_load_gap: 2,
+            })
+            .is_err()
+        );
+    }
+
     #[test]
     fn test_validate_empty_worker_urls() {
         let config = RouterConfig::new(