From 950c64545777fa67532be01ad48523ac3e6c92d7 Mon Sep 17 00:00:00 2001 From: Barry Greengus Date: Thu, 13 Aug 2026 17:26:25 +0000 Subject: [PATCH 1/5] feat(stargate): configure power-of-two sample count Allow operators to tune the number of distinct eligible backends sampled while preserving the allocation-free default path. Closes #824 --- .../crates/stargate-bench/src/main.rs | 2 +- .../stargate-bench/src/microbench/lb.rs | 127 ++++-- .../crates/stargate/src/http_proxy/trace.rs | 15 + .../stargate/src/load_balancer/config.rs | 86 +++- .../stargate/src/load_balancer/factory.rs | 4 +- .../crates/stargate/src/load_balancer/mod.rs | 7 +- .../src/load_balancer/power_of_two.rs | 413 ++++++++++++------ .../stargate/src/load_balancer/tests.rs | 98 +++++ .../docs/load-balancer-configuration.md | 34 +- 9 files changed, 611 insertions(+), 175 deletions(-) diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/main.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/main.rs index 106cb6db6..8b409eb2e 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/main.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/main.rs @@ -92,7 +92,7 @@ enum Command { Run(RunArgs), /// Compare Raw QUIC, HTTP/3, and WebTransport tunnel transports on loopback TransportBench(TransportBenchArgs), - /// Measure in-process wait-and-widen/pulsar load-balancer choose-path overhead + /// Measure in-process load-balancer choose-path overhead LbMicrobench { #[arg(long, default_value_t = 100_000, value_name = "N")] iterations: usize, diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs index 44f53e7b8..c821ada07 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs @@ -26,6 +26,7 @@ use clap::ValueEnum; use stargate::load_balancer::{ LoadBalancerAlgorithm, LoadBalancerAlgorithmConfig, LoadBalancerConfig, LoadBalancerModelConfig, LoadBalancerRequest, LoadBalancerRouter, LoadBalancerTargetState, + MAX_POWER_OF_TWO_SAMPLE_COUNT, }; use stargate::routing::{RoutedClusterSnapshot, RoutingTargetKey}; use stargate_proto::pb::{InferenceServerStatus, ModelStats}; @@ -37,27 +38,35 @@ enum WaitAndWidenTuning { Affinity, } +#[derive(Clone, Copy, PartialEq)] +enum PowerOfTwoSampleCount { + Fixed(usize), + FullPool, +} + struct LbMicrobenchScenarioMetadata { model_id: &'static str, algorithm: LoadBalancerAlgorithm, tuning: Option, + power_of_two_sample_count: Option, excluded_clusters: usize, } use WaitAndWidenTuning::{Affinity, IgnoreQueue, RttOnly}; macro_rules! scenarios { - ($($scenario:ident, $model_id:literal, $algorithm:ident, $tuning:expr, $excluded:literal;)+) => { + ($($scenario:ident, $model_id:literal, $algorithm:ident, $tuning:expr, $power_of_two_sample_count:expr, $excluded:literal;)+) => { #[repr(usize)] #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] #[value(rename_all = "kebab-case")] pub enum LbMicrobenchScenario { $($scenario,)+ } - const LB_MICROBENCH_SCENARIOS: [LbMicrobenchScenarioMetadata; 19] = [$( + const LB_MICROBENCH_SCENARIOS: [LbMicrobenchScenarioMetadata; 23] = [$( LbMicrobenchScenarioMetadata { model_id: $model_id, algorithm: LoadBalancerAlgorithm::$algorithm, tuning: $tuning, + power_of_two_sample_count: $power_of_two_sample_count, excluded_clusters: $excluded, }, )+]; @@ -67,25 +76,29 @@ macro_rules! scenarios { // Keep the scenario matrix row-oriented so differences remain directly comparable. #[rustfmt::skip] scenarios! { - PowerOfTwo, "lb-bench-power-of-two", PowerOfTwo, None, 0; - PowerOfTwoOneExcluded, "lb-bench-power-of-two-one-excluded", PowerOfTwo, None, 1; - WaitAndWiden, "lb-bench-wait-and-widen", WaitAndWiden, None, 0; - WaitAndWidenOneExcluded, "lb-bench-wait-and-widen-one-excluded", WaitAndWiden, None, 1; - WaitAndWidenIgnoreQueue, "lb-bench-wait-and-widen-ignore-queue", WaitAndWiden, Some(IgnoreQueue), 0; - WaitAndWidenIgnoreQueueOneExcluded, "lb-bench-wait-and-widen-ignore-queue-one-excluded", WaitAndWiden, Some(IgnoreQueue), 1; - WaitAndWidenIgnoreQueueMultiExcluded, "lb-bench-wait-and-widen-ignore-queue-multi-excluded", WaitAndWiden, Some(IgnoreQueue), 2; - WaitAndWidenRttOnly, "lb-bench-wait-and-widen-rtt-only", WaitAndWiden, Some(RttOnly), 0; - WaitAndWidenRttOnlyOneExcluded, "lb-bench-wait-and-widen-rtt-only-one-excluded", WaitAndWiden, Some(RttOnly), 1; - WaitAndWidenRttOnlyMultiExcluded, "lb-bench-wait-and-widen-rtt-only-multi-excluded", WaitAndWiden, Some(RttOnly), 2; - WaitAndWidenAffinity, "lb-bench-wait-and-widen-affinity", WaitAndWiden, Some(Affinity), 0; - WaitAndWidenAffinityOneExcluded, "lb-bench-wait-and-widen-affinity-one-excluded", WaitAndWiden, Some(Affinity), 1; - WaitAndWidenAffinityMultiExcluded, "lb-bench-wait-and-widen-affinity-multi-excluded", WaitAndWiden, Some(Affinity), 2; - Pulsar, "lb-bench-pulsar", Pulsar, None, 0; - PulsarOneExcluded, "lb-bench-pulsar-one-excluded", Pulsar, None, 1; - Random, "lb-bench-random", Random, None, 0; - RandomOneExcluded, "lb-bench-random-one-excluded", Random, None, 1; - RoundRobinOneExcluded, "lb-bench-round-robin-one-excluded", RoundRobin, None, 1; - RoundRobinMultiExcluded, "lb-bench-round-robin-multi-excluded", RoundRobin, None, 2; + PowerOfTwo, "lb-bench-power-of-two", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(2)), 0; + PowerOfTwoSample1, "lb-bench-power-of-two-sample-1", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(1)), 0; + PowerOfTwoSample4, "lb-bench-power-of-two-sample-4", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(4)), 0; + PowerOfTwoSample8, "lb-bench-power-of-two-sample-8", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(8)), 0; + PowerOfTwoFullPool, "lb-bench-power-of-two-full-pool", PowerOfTwo, None, Some(PowerOfTwoSampleCount::FullPool), 0; + PowerOfTwoOneExcluded, "lb-bench-power-of-two-one-excluded", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(2)), 1; + WaitAndWiden, "lb-bench-wait-and-widen", WaitAndWiden, None, None, 0; + WaitAndWidenOneExcluded, "lb-bench-wait-and-widen-one-excluded", WaitAndWiden, None, None, 1; + WaitAndWidenIgnoreQueue, "lb-bench-wait-and-widen-ignore-queue", WaitAndWiden, Some(IgnoreQueue), None, 0; + WaitAndWidenIgnoreQueueOneExcluded, "lb-bench-wait-and-widen-ignore-queue-one-excluded", WaitAndWiden, Some(IgnoreQueue), None, 1; + WaitAndWidenIgnoreQueueMultiExcluded, "lb-bench-wait-and-widen-ignore-queue-multi-excluded", WaitAndWiden, Some(IgnoreQueue), None, 2; + WaitAndWidenRttOnly, "lb-bench-wait-and-widen-rtt-only", WaitAndWiden, Some(RttOnly), None, 0; + WaitAndWidenRttOnlyOneExcluded, "lb-bench-wait-and-widen-rtt-only-one-excluded", WaitAndWiden, Some(RttOnly), None, 1; + WaitAndWidenRttOnlyMultiExcluded, "lb-bench-wait-and-widen-rtt-only-multi-excluded", WaitAndWiden, Some(RttOnly), None, 2; + WaitAndWidenAffinity, "lb-bench-wait-and-widen-affinity", WaitAndWiden, Some(Affinity), None, 0; + WaitAndWidenAffinityOneExcluded, "lb-bench-wait-and-widen-affinity-one-excluded", WaitAndWiden, Some(Affinity), None, 1; + WaitAndWidenAffinityMultiExcluded, "lb-bench-wait-and-widen-affinity-multi-excluded", WaitAndWiden, Some(Affinity), None, 2; + Pulsar, "lb-bench-pulsar", Pulsar, None, None, 0; + PulsarOneExcluded, "lb-bench-pulsar-one-excluded", Pulsar, None, None, 1; + Random, "lb-bench-random", Random, None, None, 0; + RandomOneExcluded, "lb-bench-random-one-excluded", Random, None, None, 1; + RoundRobinOneExcluded, "lb-bench-round-robin-one-excluded", RoundRobin, None, None, 1; + RoundRobinMultiExcluded, "lb-bench-round-robin-multi-excluded", RoundRobin, None, None, 2; } impl LbMicrobenchScenario { @@ -117,6 +130,7 @@ pub struct LbMicrobenchConfig { #[derive(Clone, Debug)] pub struct LbMicrobenchRow { pub scenario: LbMicrobenchScenario, + pub configured_sample_count: Option, pub candidates: usize, pub iterations: usize, pub warmup_iterations: usize, @@ -156,13 +170,15 @@ pub fn write_lb_microbench_csv( ) -> std::io::Result<()> { writeln!( writer, - "methodology,scenario,candidates,iterations,warmup_iterations,concurrency,total_ns,ns_per_choose,choices,avg_rank_depth,selected_backend_count,top_backend,top_backend_choices,backend_counts,checksum" + "methodology,scenario,configured_sample_count,candidates,iterations,warmup_iterations,concurrency,total_ns,ns_per_choose,choices,avg_rank_depth,selected_backend_count,top_backend,top_backend_choices,backend_counts,checksum" )?; for row in rows { writeln!( writer, - "choose-only-v2,{},{},{},{},{},{},{:.1},{},{:.3},{},{},{},{},{}", + "choose-only-v3,{},{},{},{},{},{},{},{:.1},{},{:.3},{},{},{},{},{}", row.scenario, + row.configured_sample_count + .map_or_else(String::new, |count| count.to_string()), row.candidates, row.iterations, row.warmup_iterations, @@ -192,6 +208,16 @@ fn validate_config(config: &LbMicrobenchConfig) -> anyhow::Result<()> { anyhow::bail!("{flag} must be greater than 0"); } } + if config.candidates > MAX_POWER_OF_TWO_SAMPLE_COUNT + && (config.scenarios.is_empty() + || config + .scenarios + .contains(&LbMicrobenchScenario::PowerOfTwoFullPool)) + { + anyhow::bail!( + "power-of-two-full-pool requires --candidates to be at most {MAX_POWER_OF_TWO_SAMPLE_COUNT}" + ); + } Ok(()) } @@ -201,12 +227,16 @@ fn run_scenario( candidates: &[RoutedClusterSnapshot], cache_keys: &[String], ) -> anyhow::Result { + let algorithm_config = config_for_scenario(scenario, config.candidates); + let configured_sample_count = algorithm_config + .power_of_two_settings() + .map(|settings| settings.sample_count()); let router = LoadBalancerRouter::from_config(&LoadBalancerConfig { default: LoadBalancerAlgorithm::PowerOfTwo, request_algorithms: HashMap::new(), models: HashMap::from([( scenario.metadata().model_id.to_string(), - LoadBalancerModelConfig::Detailed(Box::new(config_for_scenario(scenario))), + LoadBalancerModelConfig::Detailed(Box::new(algorithm_config)), )]), }) .with_context(|| format!("failed to build {scenario} load balancer"))?; @@ -258,6 +288,7 @@ fn run_scenario( let (top_backend, top_backend_choices) = top_backend(&backend_counts); Ok(LbMicrobenchRow { scenario, + configured_sample_count, candidates: config.candidates, iterations: config.iterations, warmup_iterations: config.warmup_iterations, @@ -456,12 +487,25 @@ fn format_backend_counts(backend_counts: &[(String, usize)]) -> String { .join(";") } -fn config_for_scenario(scenario: LbMicrobenchScenario) -> LoadBalancerAlgorithmConfig { +fn config_for_scenario( + scenario: LbMicrobenchScenario, + candidate_count: usize, +) -> LoadBalancerAlgorithmConfig { let metadata = scenario.metadata(); let mut config = LoadBalancerAlgorithmConfig::from(metadata.algorithm); let is_pulsar = metadata.algorithm == LoadBalancerAlgorithm::Pulsar; config.request_policy_mut().require_cache_affinity_key = is_pulsar; config.request_policy_mut().require_input_tokens = is_pulsar; + if let Some(sample_count) = metadata.power_of_two_sample_count { + let sample_count = match sample_count { + PowerOfTwoSampleCount::Fixed(count) => count, + PowerOfTwoSampleCount::FullPool => candidate_count, + }; + config + .power_of_two_settings_mut() + .expect("power-of-two scenario should expose power-of-two settings") + .sample_count = Some(sample_count); + } if is_pulsar { config .set_seed(Some("lb-microbench-seed".to_string())) @@ -565,7 +609,7 @@ mod tests { config.scenarios.clear(); let rows = run_lb_microbench(&config).expect("microbench should run"); - assert_eq!(rows.len(), 19); + assert_eq!(rows.len(), 23); assert_eq!(rows[0].scenario, LbMicrobenchScenario::PowerOfTwo); for row in rows { assert_eq!(row.choices, row.iterations); @@ -609,6 +653,29 @@ mod tests { } } + #[test] + fn power_of_two_microbench_scenarios_cover_requested_sample_counts() { + for (scenario, expected_sample_count) in [ + (LbMicrobenchScenario::PowerOfTwoSample1, 1), + (LbMicrobenchScenario::PowerOfTwo, 2), + (LbMicrobenchScenario::PowerOfTwoSample4, 4), + (LbMicrobenchScenario::PowerOfTwoSample8, 8), + (LbMicrobenchScenario::PowerOfTwoFullPool, 8), + ] { + let rows = run_lb_microbench(&config(scenario)).expect("microbench should run"); + assert_eq!(rows[0].configured_sample_count, Some(expected_sample_count)); + } + } + + #[test] + fn full_pool_microbench_rejects_candidate_count_above_sample_limit() { + let mut config = config(LbMicrobenchScenario::PowerOfTwoFullPool); + config.candidates = MAX_POWER_OF_TWO_SAMPLE_COUNT + 1; + + let error = run_lb_microbench(&config).expect_err("oversized full pool should fail"); + assert!(error.to_string().contains("power-of-two-full-pool")); + } + #[test] fn lb_microbench_rejects_zero_iterations() { let mut config = config(LbMicrobenchScenario::Pulsar); @@ -653,6 +720,7 @@ mod tests { fn lb_microbench_csv_is_parseable() { let rows = vec![LbMicrobenchRow { scenario: LbMicrobenchScenario::Pulsar, + configured_sample_count: None, candidates: 8, iterations: 10, warmup_iterations: 2, @@ -675,10 +743,13 @@ mod tests { write_lb_microbench_csv(&mut output, &rows).expect("csv should render"); let rendered = String::from_utf8(output).expect("csv should be utf8"); - assert!(rendered.starts_with("methodology,scenario,candidates,iterations")); + assert!( + rendered + .starts_with("methodology,scenario,configured_sample_count,candidates,iterations") + ); assert!(rendered.contains("selected_backend_count,top_backend,top_backend_choices")); assert!(rendered.contains( - "choose-only-v2,pulsar,8,10,2,4,1234,123.4,10,1.200,2,cluster-0001,7,cluster-0000:3;cluster-0001:7,42" + "choose-only-v3,pulsar,,8,10,2,4,1234,123.4,10,1.200,2,cluster-0001,7,cluster-0000:3;cluster-0001:7,42" )); } diff --git a/src/libraries/rust/stargate/crates/stargate/src/http_proxy/trace.rs b/src/libraries/rust/stargate/crates/stargate/src/http_proxy/trace.rs index 83699f263..987d3be8c 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/http_proxy/trace.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/http_proxy/trace.rs @@ -57,6 +57,8 @@ pub(super) fn proxy_openai_request_span(headers: &HeaderMap) -> Span { selected_inst.snapshot_age_ms = field::Empty, routing.algorithm = field::Empty, routing.num_candidates = field::Empty, + routing.sample_count_configured = field::Empty, + routing.sample_count_effective = field::Empty, routing.rank_depth = field::Empty, routing.selected_after_kv_free_tokens_skip = field::Empty, routing.retry_attempts = field::Empty, @@ -137,6 +139,7 @@ pub(super) fn record_routing_to_span(span: &Span, routing: RoutingTraceFields<'_ #[cfg(test)] mod tests { + use super::proxy_openai_request_span; use crate::telemetry::parent_context_from_headers; use axum::http::{HeaderMap, HeaderName, HeaderValue}; use opentelemetry::global; @@ -164,4 +167,16 @@ mod tests { ); assert_eq!(span_context.span_id().to_string(), "00f067aa0ba902b7"); } + + #[test] + fn proxy_span_declares_load_balancer_sample_count_fields() { + let span = proxy_openai_request_span(&HeaderMap::new()); + let fields = span + .metadata() + .expect("proxy span should have static metadata") + .fields(); + + assert!(fields.field("routing.sample_count_configured").is_some()); + assert!(fields.field("routing.sample_count_effective").is_some()); + } } diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs index 9770c2178..2356f25f3 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs @@ -194,10 +194,34 @@ pub struct WaitAndWidenAlgorithmConfig { pub ignore_input_processing_time: Option, } -#[derive(Debug, Clone, Default, PartialEq)] +pub const DEFAULT_POWER_OF_TWO_SAMPLE_COUNT: usize = 2; +pub const MAX_POWER_OF_TWO_SAMPLE_COUNT: usize = 64; + +#[derive(Debug, Clone, Default, PartialEq, Deserialize)] +pub struct PowerOfTwoAlgorithmConfig { + pub sample_count: Option, +} + +impl PowerOfTwoAlgorithmConfig { + pub fn sample_count(&self) -> usize { + self.sample_count + .unwrap_or(DEFAULT_POWER_OF_TWO_SAMPLE_COUNT) + } + + pub(crate) fn validated_sample_count(&self) -> Result { + let sample_count = self.sample_count(); + if !(1..=MAX_POWER_OF_TWO_SAMPLE_COUNT).contains(&sample_count) { + return Err(format!( + "power-of-two sample_count must be between 1 and {MAX_POWER_OF_TWO_SAMPLE_COUNT}, got {sample_count}" + )); + } + Ok(sample_count) + } +} + +#[derive(Debug, Clone, PartialEq)] pub enum LoadBalancerAlgorithmSettings { - #[default] - PowerOfTwo, + PowerOfTwo(PowerOfTwoAlgorithmConfig), WaitAndWiden(WaitAndWidenAlgorithmConfig), RoundRobin, Random, @@ -205,10 +229,16 @@ pub enum LoadBalancerAlgorithmSettings { PulsarWaitAndWiden(WaitAndWidenAlgorithmConfig), } +impl Default for LoadBalancerAlgorithmSettings { + fn default() -> Self { + Self::PowerOfTwo(PowerOfTwoAlgorithmConfig::default()) + } +} + impl LoadBalancerAlgorithmSettings { fn algorithm(&self) -> LoadBalancerAlgorithm { match self { - Self::PowerOfTwo => LoadBalancerAlgorithm::PowerOfTwo, + Self::PowerOfTwo(_) => LoadBalancerAlgorithm::PowerOfTwo, Self::WaitAndWiden(_) => LoadBalancerAlgorithm::WaitAndWiden, Self::RoundRobin => LoadBalancerAlgorithm::RoundRobin, Self::Random => LoadBalancerAlgorithm::Random, @@ -252,7 +282,7 @@ impl LoadBalancerAlgorithmConfig { LoadBalancerAlgorithmSettings::Pulsar(seed) => seed.as_deref(), LoadBalancerAlgorithmSettings::WaitAndWiden(config) | LoadBalancerAlgorithmSettings::PulsarWaitAndWiden(config) => config.seed.as_deref(), - LoadBalancerAlgorithmSettings::PowerOfTwo + LoadBalancerAlgorithmSettings::PowerOfTwo(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random => None, } @@ -274,7 +304,7 @@ impl LoadBalancerAlgorithmConfig { config.seed = seed; Ok(()) } - LoadBalancerAlgorithmSettings::PowerOfTwo + LoadBalancerAlgorithmSettings::PowerOfTwo(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random => { Err(LoadBalancerSeedError::Unsupported { algorithm }) @@ -286,7 +316,7 @@ impl LoadBalancerAlgorithmConfig { match &self.settings { LoadBalancerAlgorithmSettings::WaitAndWiden(config) | LoadBalancerAlgorithmSettings::PulsarWaitAndWiden(config) => Some(config), - LoadBalancerAlgorithmSettings::PowerOfTwo + LoadBalancerAlgorithmSettings::PowerOfTwo(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random | LoadBalancerAlgorithmSettings::Pulsar(_) => None, @@ -297,12 +327,34 @@ impl LoadBalancerAlgorithmConfig { match &mut self.settings { LoadBalancerAlgorithmSettings::WaitAndWiden(config) | LoadBalancerAlgorithmSettings::PulsarWaitAndWiden(config) => Some(config), - LoadBalancerAlgorithmSettings::PowerOfTwo + LoadBalancerAlgorithmSettings::PowerOfTwo(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random | LoadBalancerAlgorithmSettings::Pulsar(_) => None, } } + + pub fn power_of_two_settings(&self) -> Option<&PowerOfTwoAlgorithmConfig> { + match &self.settings { + LoadBalancerAlgorithmSettings::PowerOfTwo(config) => Some(config), + LoadBalancerAlgorithmSettings::WaitAndWiden(_) + | LoadBalancerAlgorithmSettings::RoundRobin + | LoadBalancerAlgorithmSettings::Random + | LoadBalancerAlgorithmSettings::Pulsar(_) + | LoadBalancerAlgorithmSettings::PulsarWaitAndWiden(_) => None, + } + } + + pub fn power_of_two_settings_mut(&mut self) -> Option<&mut PowerOfTwoAlgorithmConfig> { + match &mut self.settings { + LoadBalancerAlgorithmSettings::PowerOfTwo(config) => Some(config), + LoadBalancerAlgorithmSettings::WaitAndWiden(_) + | LoadBalancerAlgorithmSettings::RoundRobin + | LoadBalancerAlgorithmSettings::Random + | LoadBalancerAlgorithmSettings::Pulsar(_) + | LoadBalancerAlgorithmSettings::PulsarWaitAndWiden(_) => None, + } + } } impl From for LoadBalancerAlgorithmConfig { @@ -317,7 +369,7 @@ impl From for LoadBalancerAlgorithmConfig { impl From for LoadBalancerAlgorithmSettings { fn from(algorithm: LoadBalancerAlgorithm) -> Self { match algorithm { - LoadBalancerAlgorithm::PowerOfTwo => Self::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfTwo => Self::PowerOfTwo(Default::default()), LoadBalancerAlgorithm::WaitAndWiden => Self::WaitAndWiden(Default::default()), LoadBalancerAlgorithm::RoundRobin => Self::RoundRobin, LoadBalancerAlgorithm::Random => Self::Random, @@ -374,7 +426,12 @@ impl RawCommonAlgorithmConfig { #[derive(Debug, Deserialize)] #[serde(tag = "algorithm", rename_all = "kebab-case")] enum RawLoadBalancerAlgorithmConfig { - PowerOfTwo(RawCommonAlgorithmConfig), + PowerOfTwo { + #[serde(flatten)] + settings: PowerOfTwoAlgorithmConfig, + #[serde(flatten)] + common: RawCommonAlgorithmConfig, + }, #[serde(alias = "groq-multiregion")] WaitAndWiden { #[serde(flatten)] @@ -409,7 +466,11 @@ impl RawLoadBalancerAlgorithmConfig { Option, ) { match self { - Self::PowerOfTwo(common) => (common, LoadBalancerAlgorithmSettings::PowerOfTwo, None), + Self::PowerOfTwo { settings, common } => ( + common, + LoadBalancerAlgorithmSettings::PowerOfTwo(settings), + None, + ), Self::WaitAndWiden { settings, common } => ( common, LoadBalancerAlgorithmSettings::WaitAndWiden(settings), @@ -440,6 +501,9 @@ impl RawLoadBalancerAlgorithmConfig { fn into_config(self) -> Result { let (common, settings, consider_kv_free_tokens) = self.normalized(); + if let LoadBalancerAlgorithmSettings::PowerOfTwo(config) = &settings { + config.validated_sample_count()?; + } common.into_config(settings, consider_kv_free_tokens) } } diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/factory.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/factory.rs index 8dbd9fa46..321a02cb2 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/factory.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/factory.rs @@ -38,7 +38,9 @@ pub fn create_load_balancer_with_config( } match config.algorithm() { - LoadBalancerAlgorithm::PowerOfTwo => Ok(Arc::new(PowerOfTwoLoadBalancer)), + LoadBalancerAlgorithm::PowerOfTwo => Ok(Arc::new( + PowerOfTwoLoadBalancer::from_algorithm_config(config)?, + )), LoadBalancerAlgorithm::WaitAndWiden => Ok(Arc::new(WaitAndWidenLoadBalancer::new( WaitAndWidenConfig::from_algorithm_config(config), ))), diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs index 02bd8bec6..8baf21869 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs @@ -42,9 +42,10 @@ pub use algorithm::LoadBalancer; pub(crate) use algorithm::input_work_seconds_for_request; pub(super) use algorithm::{HashInputBuilder, cache_affinity_key_is_cacheable, input_work_units}; pub use config::{ - LoadBalancerAlgorithm, LoadBalancerAlgorithmConfig, LoadBalancerAlgorithmOverride, - LoadBalancerAlgorithmSettings, LoadBalancerConfig, LoadBalancerModelConfig, - LoadBalancerRequestPolicy, LoadBalancerRoutingAlgorithmError, LoadBalancerSeedError, + DEFAULT_POWER_OF_TWO_SAMPLE_COUNT, LoadBalancerAlgorithm, LoadBalancerAlgorithmConfig, + LoadBalancerAlgorithmOverride, LoadBalancerAlgorithmSettings, LoadBalancerConfig, + LoadBalancerModelConfig, LoadBalancerRequestPolicy, LoadBalancerRoutingAlgorithmError, + LoadBalancerSeedError, MAX_POWER_OF_TWO_SAMPLE_COUNT, PowerOfTwoAlgorithmConfig, WaitAndWidenAlgorithmConfig, }; pub use factory::create_load_balancer_with_config; diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_two.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_two.rs index 070aa7fad..14504842b 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_two.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_two.rs @@ -14,18 +14,49 @@ // limitations under the License. use rand::Rng; +use rand::seq::IteratorRandom; use stargate_protocol::common::valid_last_mean_input_tps; -use tracing::debug; +use tracing::{Span, debug}; #[cfg(test)] use super::tests::LoadBalancerTestChoiceExt; -use super::{LoadBalancer, LoadBalancerCandidateChoice, LoadBalancerRequest}; +use super::{ + LoadBalancer, LoadBalancerAlgorithmConfig, LoadBalancerCandidateChoice, LoadBalancerRequest, + MAX_POWER_OF_TWO_SAMPLE_COUNT, +}; use crate::routing_state::RoutedClusterSnapshot; -const EXCLUSION_REJECTION_ATTEMPTS: usize = 8; -const REJECTION_SAMPLE_EXCLUSION_RATIO_DIVISOR: usize = 4; +pub(super) struct PowerOfTwoLoadBalancer { + sample_count: usize, +} + +impl PowerOfTwoLoadBalancer { + pub(super) fn from_algorithm_config( + config: &LoadBalancerAlgorithmConfig, + ) -> anyhow::Result { + let settings = config + .power_of_two_settings() + .expect("power-of-two settings should match load-balancer algorithm"); + let sample_count = settings + .validated_sample_count() + .map_err(anyhow::Error::msg)?; + Ok(Self { sample_count }) + } + + fn choose_candidate_with_rng( + &self, + request: &LoadBalancerRequest<'_>, + candidates: &[RoutedClusterSnapshot], + rng: &mut R, + ) -> Option { + let sampled = sample_candidates(request, candidates, self.sample_count, rng); + let span = Span::current(); + span.record("routing.sample_count_configured", self.sample_count); + span.record("routing.sample_count_effective", sampled.len()); -pub(super) struct PowerOfTwoLoadBalancer; + choose_least_loaded(candidates, sampled.as_slice(), request.input_tokens, rng) + } +} impl_display!(PowerOfTwoLoadBalancer, "power-of-two"); @@ -36,157 +67,118 @@ impl LoadBalancer for PowerOfTwoLoadBalancer { candidates: &[RoutedClusterSnapshot], ) -> Option { let mut rng = rand::rng(); - let sampled = if request.has_excluded_clusters() { - sample_with_exclusions(request, candidates, &mut rng) - } else { - sample_without_exclusions(candidates, &mut rng) - }; - - match sampled { - CandidateSample::None => None, - CandidateSample::One(candidate_index) => Some( - LoadBalancerCandidateChoice::with_rank_depth_1(candidate_index), - ), - CandidateSample::Two(a_index, b_index) => Some(choose_less_loaded( - candidates, - a_index, - b_index, - request.input_tokens, - &mut rng, - )), - } + self.choose_candidate_with_rng(request, candidates, &mut rng) } } -enum CandidateSample { - None, - One(usize), - Two(usize, usize), +struct CandidateSample { + indices: [usize; MAX_POWER_OF_TWO_SAMPLE_COUNT], + len: usize, } -fn sample_without_exclusions( - candidates: &[RoutedClusterSnapshot], - rng: &mut R, -) -> CandidateSample { - match candidates.len() { - 0 => CandidateSample::None, - 1 => CandidateSample::One(0), - len => { - // First-attempt proxy routing normally has no failed clusters. Pick two - // distinct indices directly so the default production algorithm is O(1) - // instead of scanning every candidate on each request. - let (a_index, b_index) = sample_distinct_pair(len, rng); - CandidateSample::Two(a_index, b_index) +impl CandidateSample { + fn new() -> Self { + Self { + indices: [0; MAX_POWER_OF_TWO_SAMPLE_COUNT], + len: 0, } } -} -fn sample_distinct_pair(len: usize, rng: &mut R) -> (usize, usize) { - let a_index = rng.random_range(0..len); - let mut b_index = rng.random_range(0..len - 1); - if b_index >= a_index { - b_index += 1; + fn as_slice(&self) -> &[usize] { + &self.indices[..self.len] } - (a_index, b_index) -} -fn sample_with_exclusions( - request: &LoadBalancerRequest<'_>, - candidates: &[RoutedClusterSnapshot], - rng: &mut R, -) -> CandidateSample { - sample_sparse_pair(request, candidates, rng) - .unwrap_or_else(|| sample_with_reservoir(request, candidates, rng)) -} - -fn sample_sparse_pair( - request: &LoadBalancerRequest<'_>, - candidates: &[RoutedClusterSnapshot], - rng: &mut R, -) -> Option { - let excluded = request.excluded_cluster_ids?; - if candidates.len() < 2 - || excluded.len() > candidates.len() / REJECTION_SAMPLE_EXCLUSION_RATIO_DIVISOR - { - return None; - } - // Sparse retries use bounded O(1) pair sampling before uniform reservoir fallback. - for _ in 0..EXCLUSION_REJECTION_ATTEMPTS { - let (a_index, b_index) = sample_distinct_pair(candidates.len(), rng); - if !request.excludes_cluster(&candidates[a_index].cluster_id) - && !request.excludes_cluster(&candidates[b_index].cluster_id) - { - return Some(CandidateSample::Two(a_index, b_index)); - } + fn len(&self) -> usize { + self.len } - None } -fn sample_with_reservoir( +fn sample_candidates( request: &LoadBalancerRequest<'_>, candidates: &[RoutedClusterSnapshot], + sample_count: usize, rng: &mut R, ) -> CandidateSample { - let mut sampled: [Option; 2] = [None, None]; - let mut eligible_seen = 0usize; - - for (candidate_index, candidate) in candidates.iter().enumerate() { - if request.excludes_cluster(&candidate.cluster_id) { - continue; - } + let mut sampled = CandidateSample::new(); + if candidates.is_empty() { + return sampled; + } - eligible_seen += 1; - match eligible_seen { - 1 => sampled[0] = Some(candidate_index), - 2 => sampled[1] = Some(candidate_index), - _ => { - // Retry/failover attempts may exclude clusters. Reservoir sampling - // preserves the old uniform sample-without-replacement behavior for - // the remaining eligible set without allocating a filtered Vec. - let slot = rng.random_range(0..eligible_seen); - if slot < sampled.len() { - sampled[slot] = Some(candidate_index); + if !request.has_excluded_clusters() { + sampled.len = sample_count.min(candidates.len()); + match sampled.len { + 0 => {} + 1 => sampled.indices[0] = rng.random_range(0..candidates.len()), + 2 => { + let (first, second) = sample_distinct_pair(candidates.len(), rng); + sampled.indices[..2].copy_from_slice(&[first, second]); + } + len if len == candidates.len() => { + for (index, slot) in sampled.indices[..len].iter_mut().enumerate() { + *slot = index; } } + len => { + let filled = + (0..candidates.len()).choose_multiple_fill(rng, &mut sampled.indices[..len]); + debug_assert_eq!(filled, len); + } } + return sampled; } - match (sampled[0], sampled[1]) { - (None, _) => CandidateSample::None, - (Some(candidate), None) => CandidateSample::One(candidate), - (Some(a), Some(b)) => CandidateSample::Two(a, b), + let eligible_indices = candidates + .iter() + .enumerate() + .filter(|(_, candidate)| !request.excludes_cluster(&candidate.cluster_id)) + .map(|(index, _)| index); + sampled.len = eligible_indices.choose_multiple_fill(rng, &mut sampled.indices[..sample_count]); + sampled +} + +fn sample_distinct_pair(len: usize, rng: &mut R) -> (usize, usize) { + let a_index = rng.random_range(0..len); + let mut b_index = rng.random_range(0..len - 1); + if b_index >= a_index { + b_index += 1; } + (a_index, b_index) } -fn choose_less_loaded( +fn choose_least_loaded( candidates: &[RoutedClusterSnapshot], - a_index: usize, - b_index: usize, + sampled_indices: &[usize], input_tokens: Option, rng: &mut R, -) -> LoadBalancerCandidateChoice { - let a = &candidates[a_index]; - let b = &candidates[b_index]; - let a_score = load_score(a, input_tokens); - let b_score = load_score(b, input_tokens); +) -> Option { + let (&first_index, remaining_indices) = sampled_indices.split_first()?; + let mut selected_index = first_index; + let mut selected_score = load_score(&candidates[first_index], input_tokens); + let mut tied_best_count = 1u32; + + for &candidate_index in remaining_indices { + let score = load_score(&candidates[candidate_index], input_tokens); + if score < selected_score { + selected_index = candidate_index; + selected_score = score; + tied_best_count = 1; + } else if score == selected_score { + tied_best_count += 1; + if rng.random_ratio(1, tied_best_count) { + selected_index = candidate_index; + } + } + } + debug!( - inst_a = %a.cluster_id, - inst_b = %b.cluster_id, - load_score_a = a_score, - load_score_b = b_score, - "sampled two clusters" + effective_sample_count = sampled_indices.len(), + selected_candidate_index = selected_index, + selected_load_score = selected_score, + "sampled clusters" ); - - let selected_index = if a_score < b_score { - a_index - } else if b_score < a_score { - b_index - } else if rng.random_bool(0.5) { - a_index - } else { - b_index - }; - LoadBalancerCandidateChoice::with_rank_depth_1(selected_index) + Some(LoadBalancerCandidateChoice::with_rank_depth_1( + selected_index, + )) } fn load_score(candidate: &RoutedClusterSnapshot, input_tokens: Option) -> f64 { @@ -204,6 +196,8 @@ mod tests { use std::collections::HashSet; use std::time::{Duration, Instant}; + use rand::SeedableRng; + use rand::rngs::StdRng; use stargate_proto::pb::{InferenceServerStatus, ModelStats}; use super::*; @@ -228,6 +222,7 @@ mod tests { } fn selected_cluster_id( + sample_count: usize, candidates: &[RoutedClusterSnapshot], excluded_cluster_ids: &HashSet, ) -> Option { @@ -241,11 +236,32 @@ mod tests { request_slo: None, excluded_cluster_ids: Some(excluded_cluster_ids), }; - PowerOfTwoLoadBalancer + PowerOfTwoLoadBalancer { sample_count } .choose_for_test(&request, candidates) .map(|choice| choice.candidate.cluster_id) } + fn sampled_indices( + sample_count: usize, + candidates: &[RoutedClusterSnapshot], + excluded_cluster_ids: Option<&HashSet>, + rng: &mut StdRng, + ) -> Vec { + let target = crate::routing_state::RoutingTargetKey::new(None, "model-a"); + let request = LoadBalancerRequest { + routing_target: &target, + cache_affinity_key: None, + input_tokens: Some(1000), + priority: 0, + received_at: Instant::now(), + request_slo: None, + excluded_cluster_ids, + }; + sample_candidates(&request, candidates, sample_count, rng) + .as_slice() + .to_vec() + } + #[test] fn load_score_prefers_faster_empty_backend_for_incoming_prefill() { let fast = candidate("fast", 200.0, 0); @@ -280,7 +296,7 @@ mod tests { for _ in 0..64 { assert_eq!( - selected_cluster_id(&candidates, &excluded).as_deref(), + selected_cluster_id(2, &candidates, &excluded).as_deref(), Some("eligible") ); } @@ -294,7 +310,7 @@ mod tests { let excluded = HashSet::from(["cluster-0000".to_string()]); for _ in 0..512 { - let selected = selected_cluster_id(&candidates, &excluded) + let selected = selected_cluster_id(2, &candidates, &excluded) .expect("an eligible cluster should be selected"); assert_ne!(selected, "cluster-0000"); } @@ -308,6 +324,145 @@ mod tests { ]; let excluded = HashSet::from(["excluded-a".to_string(), "excluded-b".to_string()]); - assert!(selected_cluster_id(&candidates, &excluded).is_none()); + assert!(selected_cluster_id(2, &candidates, &excluded).is_none()); + } + + #[test] + fn sampling_handles_empty_singleton_and_oversized_pools() { + let one = vec![candidate("only", 1.0, 0)]; + let three = vec![ + candidate("a", 1.0, 0), + candidate("b", 1.0, 0), + candidate("c", 1.0, 0), + ]; + let mut rng = StdRng::seed_from_u64(7); + + assert!(sampled_indices(4, &[], None, &mut rng).is_empty()); + assert_eq!(sampled_indices(4, &one, None, &mut rng), [0]); + assert_eq!(sampled_indices(3, &three, None, &mut rng), [0, 1, 2]); + assert_eq!(sampled_indices(8, &three, None, &mut rng), [0, 1, 2]); + } + + #[test] + fn sampling_is_distinct_and_reproducible_for_counts_one_two_and_four() { + let candidates = (0..8) + .map(|index| candidate(&format!("cluster-{index}"), 1.0, 0)) + .collect::>(); + + for sample_count in [1, 2, 4] { + let mut first_rng = StdRng::seed_from_u64(42); + let mut second_rng = StdRng::seed_from_u64(42); + let first = sampled_indices(sample_count, &candidates, None, &mut first_rng); + let second = sampled_indices(sample_count, &candidates, None, &mut second_rng); + let distinct = first.iter().copied().collect::>(); + + assert_eq!(first, second); + assert_eq!(first.len(), sample_count); + assert_eq!(distinct.len(), sample_count); + } + } + + #[test] + fn sampling_applies_sparse_and_dense_exclusions_before_sampling() { + let candidates = (0..8) + .map(|index| candidate(&format!("cluster-{index}"), 1.0, 0)) + .collect::>(); + + for excluded in [ + HashSet::from(["cluster-0".to_string()]), + HashSet::from([ + "cluster-0".to_string(), + "cluster-1".to_string(), + "cluster-2".to_string(), + "cluster-3".to_string(), + "cluster-4".to_string(), + "cluster-5".to_string(), + ]), + ] { + let mut rng = StdRng::seed_from_u64(91); + let sampled = sampled_indices(4, &candidates, Some(&excluded), &mut rng); + let expected_count = 4.min(candidates.len() - excluded.len()); + + assert_eq!(sampled.len(), expected_count); + assert!( + sampled + .iter() + .all(|index| !excluded.contains(&candidates[*index].cluster_id)) + ); + assert_eq!( + sampled.iter().copied().collect::>().len(), + expected_count + ); + } + } + + #[test] + fn sampling_is_uniform_without_replacement() { + let candidates = (0..4) + .map(|index| candidate(&format!("cluster-{index}"), 1.0, 0)) + .collect::>(); + let mut rng = StdRng::seed_from_u64(73); + let mut inclusion_counts = [0usize; 4]; + const ITERATIONS: usize = 20_000; + + for _ in 0..ITERATIONS { + for index in sampled_indices(2, &candidates, None, &mut rng) { + inclusion_counts[index] += 1; + } + } + + let expected = ITERATIONS / 2; + let tolerance = expected / 25; + for count in inclusion_counts { + assert!(count.abs_diff(expected) <= tolerance, "count={count}"); + } + } + + #[test] + fn full_pool_selection_chooses_the_lowest_score() { + let candidates = vec![ + candidate("slow", 100.0, 1000), + candidate("best", 1000.0, 0), + candidate("busy", 1000.0, 10_000), + ]; + let excluded = HashSet::new(); + + assert_eq!( + selected_cluster_id(3, &candidates, &excluded).as_deref(), + Some("best") + ); + } + + #[test] + fn equal_scores_do_not_bias_ties_to_one_candidate() { + let candidates = vec![ + candidate("a", 100.0, 0), + candidate("b", 100.0, 0), + candidate("c", 100.0, 0), + ]; + let target = crate::routing_state::RoutingTargetKey::new(None, "model-a"); + let request = LoadBalancerRequest { + routing_target: &target, + cache_affinity_key: None, + input_tokens: Some(1000), + priority: 0, + received_at: Instant::now(), + request_slo: None, + excluded_cluster_ids: None, + }; + let load_balancer = PowerOfTwoLoadBalancer { sample_count: 3 }; + let mut rng = StdRng::seed_from_u64(11); + let mut selected = HashSet::new(); + + for _ in 0..64 { + selected.insert( + load_balancer + .choose_candidate_with_rng(&request, &candidates, &mut rng) + .expect("equal-score pool should produce a candidate") + .candidate_index, + ); + } + + assert_eq!(selected, HashSet::from([0, 1, 2])); } } diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs index 3ba3c16e3..dfef64464 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs @@ -714,11 +714,109 @@ fn algorithm_specific_load_balancer_fields_are_rejected_for_other_algorithms() { r#"{"algorithm":"wait-and-widen","consider_kv_free_tokens":true}"#, "consider_kv_free_tokens", ), + (r#"{"algorithm":"random","sample_count":4}"#, "sample_count"), ] { assert_json_rejected::(raw, expected_field); } } +#[test] +fn power_of_two_sample_count_defaults_to_two() { + let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfTwo); + let settings = config + .power_of_two_settings() + .expect("power-of-two config should expose settings"); + + assert_eq!(settings.sample_count(), DEFAULT_POWER_OF_TWO_SAMPLE_COUNT); +} + +#[test] +fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { + let direct: LoadBalancerAlgorithmConfig = + parse_json(r#"{"algorithm":"power-of-two","sample_count":1}"#); + assert_eq!( + direct + .power_of_two_settings() + .expect("direct config should expose settings") + .sample_count(), + 1 + ); + + let router = router_from_json( + r#"{"default":"random","request_algorithms":{"power-of-two":{"algorithm":"power-of-two","sample_count":4}},"models":{"model-a":{"algorithm":"power-of-two","sample_count":8,"request_algorithms":{"power-of-two":{"algorithm":"power-of-two","sample_count":64}}}}}"#, + ); + assert_eq!( + router + .algorithm_config("model-a") + .power_of_two_settings() + .expect("model config should expose settings") + .sample_count(), + 8 + ); + + let override_header = LoadBalancerAlgorithmOverride::parse("power-of-two") + .expect("power-of-two override should parse"); + let model_override = router + .resolve_algorithm_override("model-a", Some(&override_header)) + .expect("model override should resolve"); + let default_override = router + .resolve_algorithm_override("model-b", Some(&override_header)) + .expect("top-level override should resolve"); + assert_eq!( + model_override + .config() + .power_of_two_settings() + .expect("model override should expose settings") + .sample_count(), + 8, + "the configured model algorithm takes precedence over its same-algorithm override" + ); + assert_eq!( + default_override + .config() + .power_of_two_settings() + .expect("top-level override should expose settings") + .sample_count(), + 4 + ); + + let nested_router = router_from_json( + r#"{"default":"random","models":{"model-a":{"algorithm":"random","request_algorithms":{"power-of-two":{"algorithm":"power-of-two","sample_count":64}}}}}"#, + ); + let nested_override = nested_router + .resolve_algorithm_override("model-a", Some(&override_header)) + .expect("nested override should resolve"); + assert_eq!( + nested_override + .config() + .power_of_two_settings() + .expect("nested override should expose settings") + .sample_count(), + 64 + ); +} + +#[test] +fn invalid_power_of_two_sample_counts_are_rejected_with_field_context() { + for sample_count in [0, MAX_POWER_OF_TWO_SAMPLE_COUNT + 1] { + assert_json_rejected::( + &format!(r#"{{"algorithm":"power-of-two","sample_count":{sample_count}}}"#), + "power-of-two sample_count must be between 1 and 64", + ); + } + + let mut config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfTwo); + config + .power_of_two_settings_mut() + .expect("power-of-two config should expose mutable settings") + .sample_count = Some(0); + let error = match create_load_balancer_with_config(&config) { + Ok(_) => panic!("programmatic invalid sample count should fail"), + Err(error) => error, + }; + assert!(error.to_string().contains("power-of-two sample_count")); +} + #[test] fn detailed_algorithm_configs_preserve_all_variant_identities() { use LoadBalancerAlgorithm::*; diff --git a/src/libraries/rust/stargate/docs/load-balancer-configuration.md b/src/libraries/rust/stargate/docs/load-balancer-configuration.md index e2b3f8979..de8329e8f 100644 --- a/src/libraries/rust/stargate/docs/load-balancer-configuration.md +++ b/src/libraries/rust/stargate/docs/load-balancer-configuration.md @@ -4,8 +4,8 @@ Stargate selects one load-balancing algorithm for each model. A request can select another preconfigured algorithm through a trusted header. This page defines the `lb-config.json` schema and the behavior of -`wait-and-widen`, `pulsar`, and `pulsar-wait-and-widen`. Deployment systems own -the file mount and the `--lb-config-path` argument. +`power-of-two`, `wait-and-widen`, `pulsar`, and `pulsar-wait-and-widen`. +Deployment systems own the file mount and the `--lb-config-path` argument. ## Load the configuration @@ -115,6 +115,30 @@ Use `power-of-two` when these statistics or affinity requirements are not available. Use `round-robin` for deterministic cycling and `random` for uniform random selection. +## `power-of-two` + +`power-of-two` uniformly samples distinct eligible clusters and selects the +cluster with the lowest sum of queued and request input tokens divided by its +last mean input TPS. It breaks equal scores randomly. Retried clusters are +excluded before sampling. + +The default sample count is `2`. A larger sample can improve routing decisions +in a heterogeneous pool, but it compares more clusters on every request. Valid +values are `1` through `64`. If fewer eligible clusters exist, the algorithm +compares every eligible cluster once. + +```json +{ + "default": "power-of-two", + "models": { + "model-a": { + "algorithm": "power-of-two", + "sample_count": 4 + } + } +} +``` + ## `wait-and-widen` `wait-and-widen` estimates time to first token (TTFT) as: @@ -209,6 +233,12 @@ Minimal configuration: ## Algorithm fields +`power-of-two` supports this field: + +| Field | Type | Default | Constraint and effect | +| --- | --- | --- | --- | +| `sample_count` | unsigned integer | `2` | Number of distinct eligible clusters sampled. Must be from `1` through `64`. Values above the eligible cluster count compare the complete eligible pool. | + `wait-and-widen` supports these cache-affinity fields: | Field | Type | Default | Constraint and effect | From 4da0588c46ebbef34876680ff2b39f4a512109ae Mon Sep 17 00:00:00 2001 From: Barry Greengus Date: Thu, 13 Aug 2026 17:42:50 +0000 Subject: [PATCH 2/5] refactor(stargate): simplify power-of-two configuration Canonicalize the configured sample count and derive benchmark metadata from the scenario instead of storing parallel state. --- .../stargate-bench/src/microbench/lb.rs | 89 +++++++++---------- .../stargate/src/load_balancer/config.rs | 20 +++-- .../crates/stargate/src/load_balancer/mod.rs | 9 +- .../stargate/src/load_balancer/tests.rs | 14 +-- 4 files changed, 66 insertions(+), 66 deletions(-) diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs index c821ada07..e22443f81 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs @@ -38,24 +38,17 @@ enum WaitAndWidenTuning { Affinity, } -#[derive(Clone, Copy, PartialEq)] -enum PowerOfTwoSampleCount { - Fixed(usize), - FullPool, -} - struct LbMicrobenchScenarioMetadata { model_id: &'static str, algorithm: LoadBalancerAlgorithm, tuning: Option, - power_of_two_sample_count: Option, excluded_clusters: usize, } use WaitAndWidenTuning::{Affinity, IgnoreQueue, RttOnly}; macro_rules! scenarios { - ($($scenario:ident, $model_id:literal, $algorithm:ident, $tuning:expr, $power_of_two_sample_count:expr, $excluded:literal;)+) => { + ($($scenario:ident, $model_id:literal, $algorithm:ident, $tuning:expr, $excluded:literal;)+) => { #[repr(usize)] #[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] #[value(rename_all = "kebab-case")] @@ -66,7 +59,6 @@ macro_rules! scenarios { model_id: $model_id, algorithm: LoadBalancerAlgorithm::$algorithm, tuning: $tuning, - power_of_two_sample_count: $power_of_two_sample_count, excluded_clusters: $excluded, }, )+]; @@ -76,35 +68,46 @@ macro_rules! scenarios { // Keep the scenario matrix row-oriented so differences remain directly comparable. #[rustfmt::skip] scenarios! { - PowerOfTwo, "lb-bench-power-of-two", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(2)), 0; - PowerOfTwoSample1, "lb-bench-power-of-two-sample-1", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(1)), 0; - PowerOfTwoSample4, "lb-bench-power-of-two-sample-4", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(4)), 0; - PowerOfTwoSample8, "lb-bench-power-of-two-sample-8", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(8)), 0; - PowerOfTwoFullPool, "lb-bench-power-of-two-full-pool", PowerOfTwo, None, Some(PowerOfTwoSampleCount::FullPool), 0; - PowerOfTwoOneExcluded, "lb-bench-power-of-two-one-excluded", PowerOfTwo, None, Some(PowerOfTwoSampleCount::Fixed(2)), 1; - WaitAndWiden, "lb-bench-wait-and-widen", WaitAndWiden, None, None, 0; - WaitAndWidenOneExcluded, "lb-bench-wait-and-widen-one-excluded", WaitAndWiden, None, None, 1; - WaitAndWidenIgnoreQueue, "lb-bench-wait-and-widen-ignore-queue", WaitAndWiden, Some(IgnoreQueue), None, 0; - WaitAndWidenIgnoreQueueOneExcluded, "lb-bench-wait-and-widen-ignore-queue-one-excluded", WaitAndWiden, Some(IgnoreQueue), None, 1; - WaitAndWidenIgnoreQueueMultiExcluded, "lb-bench-wait-and-widen-ignore-queue-multi-excluded", WaitAndWiden, Some(IgnoreQueue), None, 2; - WaitAndWidenRttOnly, "lb-bench-wait-and-widen-rtt-only", WaitAndWiden, Some(RttOnly), None, 0; - WaitAndWidenRttOnlyOneExcluded, "lb-bench-wait-and-widen-rtt-only-one-excluded", WaitAndWiden, Some(RttOnly), None, 1; - WaitAndWidenRttOnlyMultiExcluded, "lb-bench-wait-and-widen-rtt-only-multi-excluded", WaitAndWiden, Some(RttOnly), None, 2; - WaitAndWidenAffinity, "lb-bench-wait-and-widen-affinity", WaitAndWiden, Some(Affinity), None, 0; - WaitAndWidenAffinityOneExcluded, "lb-bench-wait-and-widen-affinity-one-excluded", WaitAndWiden, Some(Affinity), None, 1; - WaitAndWidenAffinityMultiExcluded, "lb-bench-wait-and-widen-affinity-multi-excluded", WaitAndWiden, Some(Affinity), None, 2; - Pulsar, "lb-bench-pulsar", Pulsar, None, None, 0; - PulsarOneExcluded, "lb-bench-pulsar-one-excluded", Pulsar, None, None, 1; - Random, "lb-bench-random", Random, None, None, 0; - RandomOneExcluded, "lb-bench-random-one-excluded", Random, None, None, 1; - RoundRobinOneExcluded, "lb-bench-round-robin-one-excluded", RoundRobin, None, None, 1; - RoundRobinMultiExcluded, "lb-bench-round-robin-multi-excluded", RoundRobin, None, None, 2; + PowerOfTwo, "lb-bench-power-of-two", PowerOfTwo, None, 0; + PowerOfTwoSample1, "lb-bench-power-of-two-sample-1", PowerOfTwo, None, 0; + PowerOfTwoSample4, "lb-bench-power-of-two-sample-4", PowerOfTwo, None, 0; + PowerOfTwoSample8, "lb-bench-power-of-two-sample-8", PowerOfTwo, None, 0; + PowerOfTwoFullPool, "lb-bench-power-of-two-full-pool", PowerOfTwo, None, 0; + PowerOfTwoOneExcluded, "lb-bench-power-of-two-one-excluded", PowerOfTwo, None, 1; + WaitAndWiden, "lb-bench-wait-and-widen", WaitAndWiden, None, 0; + WaitAndWidenOneExcluded, "lb-bench-wait-and-widen-one-excluded", WaitAndWiden, None, 1; + WaitAndWidenIgnoreQueue, "lb-bench-wait-and-widen-ignore-queue", WaitAndWiden, Some(IgnoreQueue), 0; + WaitAndWidenIgnoreQueueOneExcluded, "lb-bench-wait-and-widen-ignore-queue-one-excluded", WaitAndWiden, Some(IgnoreQueue), 1; + WaitAndWidenIgnoreQueueMultiExcluded, "lb-bench-wait-and-widen-ignore-queue-multi-excluded", WaitAndWiden, Some(IgnoreQueue), 2; + WaitAndWidenRttOnly, "lb-bench-wait-and-widen-rtt-only", WaitAndWiden, Some(RttOnly), 0; + WaitAndWidenRttOnlyOneExcluded, "lb-bench-wait-and-widen-rtt-only-one-excluded", WaitAndWiden, Some(RttOnly), 1; + WaitAndWidenRttOnlyMultiExcluded, "lb-bench-wait-and-widen-rtt-only-multi-excluded", WaitAndWiden, Some(RttOnly), 2; + WaitAndWidenAffinity, "lb-bench-wait-and-widen-affinity", WaitAndWiden, Some(Affinity), 0; + WaitAndWidenAffinityOneExcluded, "lb-bench-wait-and-widen-affinity-one-excluded", WaitAndWiden, Some(Affinity), 1; + WaitAndWidenAffinityMultiExcluded, "lb-bench-wait-and-widen-affinity-multi-excluded", WaitAndWiden, Some(Affinity), 2; + Pulsar, "lb-bench-pulsar", Pulsar, None, 0; + PulsarOneExcluded, "lb-bench-pulsar-one-excluded", Pulsar, None, 1; + Random, "lb-bench-random", Random, None, 0; + RandomOneExcluded, "lb-bench-random-one-excluded", Random, None, 1; + RoundRobinOneExcluded, "lb-bench-round-robin-one-excluded", RoundRobin, None, 1; + RoundRobinMultiExcluded, "lb-bench-round-robin-multi-excluded", RoundRobin, None, 2; } impl LbMicrobenchScenario { fn metadata(self) -> &'static LbMicrobenchScenarioMetadata { &LB_MICROBENCH_SCENARIOS[self as usize] } + + fn configured_sample_count(self, candidate_count: usize) -> Option { + match self { + Self::PowerOfTwo | Self::PowerOfTwoOneExcluded => Some(2), + Self::PowerOfTwoSample1 => Some(1), + Self::PowerOfTwoSample4 => Some(4), + Self::PowerOfTwoSample8 => Some(8), + Self::PowerOfTwoFullPool => Some(candidate_count), + _ => None, + } + } } impl fmt::Display for LbMicrobenchScenario { @@ -130,7 +133,6 @@ pub struct LbMicrobenchConfig { #[derive(Clone, Debug)] pub struct LbMicrobenchRow { pub scenario: LbMicrobenchScenario, - pub configured_sample_count: Option, pub candidates: usize, pub iterations: usize, pub warmup_iterations: usize, @@ -177,7 +179,8 @@ pub fn write_lb_microbench_csv( writer, "choose-only-v3,{},{},{},{},{},{},{},{:.1},{},{:.3},{},{},{},{},{}", row.scenario, - row.configured_sample_count + row.scenario + .configured_sample_count(row.candidates) .map_or_else(String::new, |count| count.to_string()), row.candidates, row.iterations, @@ -228,9 +231,6 @@ fn run_scenario( cache_keys: &[String], ) -> anyhow::Result { let algorithm_config = config_for_scenario(scenario, config.candidates); - let configured_sample_count = algorithm_config - .power_of_two_settings() - .map(|settings| settings.sample_count()); let router = LoadBalancerRouter::from_config(&LoadBalancerConfig { default: LoadBalancerAlgorithm::PowerOfTwo, request_algorithms: HashMap::new(), @@ -288,7 +288,6 @@ fn run_scenario( let (top_backend, top_backend_choices) = top_backend(&backend_counts); Ok(LbMicrobenchRow { scenario, - configured_sample_count, candidates: config.candidates, iterations: config.iterations, warmup_iterations: config.warmup_iterations, @@ -496,15 +495,11 @@ fn config_for_scenario( let is_pulsar = metadata.algorithm == LoadBalancerAlgorithm::Pulsar; config.request_policy_mut().require_cache_affinity_key = is_pulsar; config.request_policy_mut().require_input_tokens = is_pulsar; - if let Some(sample_count) = metadata.power_of_two_sample_count { - let sample_count = match sample_count { - PowerOfTwoSampleCount::Fixed(count) => count, - PowerOfTwoSampleCount::FullPool => candidate_count, - }; + if let Some(sample_count) = scenario.configured_sample_count(candidate_count) { config .power_of_two_settings_mut() .expect("power-of-two scenario should expose power-of-two settings") - .sample_count = Some(sample_count); + .sample_count = sample_count; } if is_pulsar { config @@ -663,7 +658,10 @@ mod tests { (LbMicrobenchScenario::PowerOfTwoFullPool, 8), ] { let rows = run_lb_microbench(&config(scenario)).expect("microbench should run"); - assert_eq!(rows[0].configured_sample_count, Some(expected_sample_count)); + assert_eq!( + rows[0].scenario.configured_sample_count(rows[0].candidates), + Some(expected_sample_count) + ); } } @@ -720,7 +718,6 @@ mod tests { fn lb_microbench_csv_is_parseable() { let rows = vec![LbMicrobenchRow { scenario: LbMicrobenchScenario::Pulsar, - configured_sample_count: None, candidates: 8, iterations: 10, warmup_iterations: 2, diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs index 2356f25f3..e26618e9c 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs @@ -194,22 +194,26 @@ pub struct WaitAndWidenAlgorithmConfig { pub ignore_input_processing_time: Option, } -pub const DEFAULT_POWER_OF_TWO_SAMPLE_COUNT: usize = 2; +const DEFAULT_POWER_OF_TWO_SAMPLE_COUNT: usize = 2; pub const MAX_POWER_OF_TWO_SAMPLE_COUNT: usize = 64; -#[derive(Debug, Clone, Default, PartialEq, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize)] +#[serde(default)] pub struct PowerOfTwoAlgorithmConfig { - pub sample_count: Option, + pub sample_count: usize, } -impl PowerOfTwoAlgorithmConfig { - pub fn sample_count(&self) -> usize { - self.sample_count - .unwrap_or(DEFAULT_POWER_OF_TWO_SAMPLE_COUNT) +impl Default for PowerOfTwoAlgorithmConfig { + fn default() -> Self { + Self { + sample_count: DEFAULT_POWER_OF_TWO_SAMPLE_COUNT, + } } +} +impl PowerOfTwoAlgorithmConfig { pub(crate) fn validated_sample_count(&self) -> Result { - let sample_count = self.sample_count(); + let sample_count = self.sample_count; if !(1..=MAX_POWER_OF_TWO_SAMPLE_COUNT).contains(&sample_count) { return Err(format!( "power-of-two sample_count must be between 1 and {MAX_POWER_OF_TWO_SAMPLE_COUNT}, got {sample_count}" diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs index 8baf21869..7618ffe77 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs @@ -42,11 +42,10 @@ pub use algorithm::LoadBalancer; pub(crate) use algorithm::input_work_seconds_for_request; pub(super) use algorithm::{HashInputBuilder, cache_affinity_key_is_cacheable, input_work_units}; pub use config::{ - DEFAULT_POWER_OF_TWO_SAMPLE_COUNT, LoadBalancerAlgorithm, LoadBalancerAlgorithmConfig, - LoadBalancerAlgorithmOverride, LoadBalancerAlgorithmSettings, LoadBalancerConfig, - LoadBalancerModelConfig, LoadBalancerRequestPolicy, LoadBalancerRoutingAlgorithmError, - LoadBalancerSeedError, MAX_POWER_OF_TWO_SAMPLE_COUNT, PowerOfTwoAlgorithmConfig, - WaitAndWidenAlgorithmConfig, + LoadBalancerAlgorithm, LoadBalancerAlgorithmConfig, LoadBalancerAlgorithmOverride, + LoadBalancerAlgorithmSettings, LoadBalancerConfig, LoadBalancerModelConfig, + LoadBalancerRequestPolicy, LoadBalancerRoutingAlgorithmError, LoadBalancerSeedError, + MAX_POWER_OF_TWO_SAMPLE_COUNT, PowerOfTwoAlgorithmConfig, WaitAndWidenAlgorithmConfig, }; pub use factory::create_load_balancer_with_config; pub use request::{LoadBalancerCandidateChoice, LoadBalancerRequest}; diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs index dfef64464..64b413720 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs @@ -727,7 +727,7 @@ fn power_of_two_sample_count_defaults_to_two() { .power_of_two_settings() .expect("power-of-two config should expose settings"); - assert_eq!(settings.sample_count(), DEFAULT_POWER_OF_TWO_SAMPLE_COUNT); + assert_eq!(settings.sample_count, 2); } #[test] @@ -738,7 +738,7 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { direct .power_of_two_settings() .expect("direct config should expose settings") - .sample_count(), + .sample_count, 1 ); @@ -750,7 +750,7 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { .algorithm_config("model-a") .power_of_two_settings() .expect("model config should expose settings") - .sample_count(), + .sample_count, 8 ); @@ -767,7 +767,7 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { .config() .power_of_two_settings() .expect("model override should expose settings") - .sample_count(), + .sample_count, 8, "the configured model algorithm takes precedence over its same-algorithm override" ); @@ -776,7 +776,7 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { .config() .power_of_two_settings() .expect("top-level override should expose settings") - .sample_count(), + .sample_count, 4 ); @@ -791,7 +791,7 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { .config() .power_of_two_settings() .expect("nested override should expose settings") - .sample_count(), + .sample_count, 64 ); } @@ -809,7 +809,7 @@ fn invalid_power_of_two_sample_counts_are_rejected_with_field_context() { config .power_of_two_settings_mut() .expect("power-of-two config should expose mutable settings") - .sample_count = Some(0); + .sample_count = 0; let error = match create_load_balancer_with_config(&config) { Ok(_) => panic!("programmatic invalid sample count should fail"), Err(error) => error, From c522d9760d1551fc4ece894acbda1b1bdbc87557 Mon Sep 17 00:00:00 2001 From: Barry Greengus Date: Thu, 13 Aug 2026 18:47:31 +0000 Subject: [PATCH 3/5] refactor(stargate): rename power-of-two to power-of-n Use power-of-n as the canonical configurable algorithm name while accepting power-of-two, powerOf2, and powerOfN at existing configuration and override boundaries. --- .../stargate/benches/backend-degradation.yaml | 6 +- .../stargate/benches/bursty-8-backends.yaml | 4 +- .../benches/cache-thrash-6-backends.yaml | 12 +- .../benches/hotset-8-backends-long.yaml | 6 +- .../stargate/benches/hotset-8-backends.yaml | 6 +- .../benches/lb-balance-bursty-4c2p-2s.yaml | 6 +- .../benches/lb-balance-hotset-8c2p-4s.yaml | 6 +- .../lb-balance-prefix-reuse-4c2p-2s.yaml | 14 +- ...use-pulsar-wait-and-widen-slo-4c2p-1s.yaml | 6 +- ...lb-balance-prefix-reuse-smoke-2c2p-1s.yaml | 14 +- .../benches/lb-balance-smoke-2c2p-1s.yaml | 6 +- .../stargate/benches/mixed-size-pulsar.yaml | 6 +- .../stargate/benches/overload-6-backends.yaml | 6 +- .../benches/stair-step-2-stargates.yaml | 6 +- .../stargate/benches/sticky-hot-prefix.yaml | 6 +- .../stargate/benches/uniform-4-backends.yaml | 4 +- .../crates/stargate-bench/src/k8s/tests.rs | 24 +-- .../crates/stargate-bench/src/k8s_run.rs | 16 +- .../crates/stargate-bench/src/main.rs | 22 +-- .../crates/stargate-bench/src/manifest.rs | 4 +- .../stargate-bench/src/microbench/lb.rs | 60 +++---- .../crates/stargate-bench/src/orchestrator.rs | 6 +- .../crates/stargate-bench/src/report.rs | 6 +- .../crates/stargate/src/http_proxy/request.rs | 2 +- .../crates/stargate/src/http_proxy/routing.rs | 6 +- .../crates/stargate/src/http_proxy/run.rs | 4 +- .../stargate/src/load_balancer/config.rs | 66 ++++---- .../stargate/src/load_balancer/factory.rs | 6 +- .../crates/stargate/src/load_balancer/mod.rs | 4 +- .../{power_of_two.rs => power_of_n.rs} | 28 ++-- .../stargate/src/load_balancer/tests.rs | 151 ++++++++++-------- .../rust/stargate/crates/stargate/src/main.rs | 2 +- .../stargate/src/routing_state/tests.rs | 2 +- .../stargate/tests/suite/integration.rs | 2 +- .../stargate/tests/suite/load_balancing.rs | 20 ++- .../stargate/tests/suite/proxy_contract.rs | 6 +- .../docs/diagrams/chat-completions-e2e.puml | 2 +- .../docs/load-balancer-configuration.md | 43 ++--- .../stargate/docs/multi-backend-clusters.md | 2 +- 39 files changed, 309 insertions(+), 289 deletions(-) rename src/libraries/rust/stargate/crates/stargate/src/load_balancer/{power_of_two.rs => power_of_n.rs} (95%) diff --git a/src/libraries/rust/stargate/benches/backend-degradation.yaml b/src/libraries/rust/stargate/benches/backend-degradation.yaml index 26e970f8d..c17ff479a 100644 --- a/src/libraries/rust/stargate/benches/backend-degradation.yaml +++ b/src/libraries/rust/stargate/benches/backend-degradation.yaml @@ -99,15 +99,15 @@ traffic_pattern: target_rps: 6 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: wait-and-widen config: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/bursty-8-backends.yaml b/src/libraries/rust/stargate/benches/bursty-8-backends.yaml index 00db155c1..21d0ce1e9 100644 --- a/src/libraries/rust/stargate/benches/bursty-8-backends.yaml +++ b/src/libraries/rust/stargate/benches/bursty-8-backends.yaml @@ -105,9 +105,9 @@ traffic_pattern: burst_period_requests: 16 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: random config: default: random diff --git a/src/libraries/rust/stargate/benches/cache-thrash-6-backends.yaml b/src/libraries/rust/stargate/benches/cache-thrash-6-backends.yaml index 463b79ebd..482895b8f 100644 --- a/src/libraries/rust/stargate/benches/cache-thrash-6-backends.yaml +++ b/src/libraries/rust/stargate/benches/cache-thrash-6-backends.yaml @@ -64,15 +64,15 @@ traffic_pattern: target_rps: 8 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: wait-and-widen config: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar @@ -81,7 +81,7 @@ algorithms: require_input_tokens: true - name: pulsar-consider-kv-free-tokens config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar @@ -90,7 +90,7 @@ algorithms: consider_kv_free_tokens: true - name: pulsar-wait-and-widen config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar-wait-and-widen @@ -99,7 +99,7 @@ algorithms: require_input_tokens: true - name: pulsar-wait-and-widen-consider-kv-free-tokens config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar-wait-and-widen diff --git a/src/libraries/rust/stargate/benches/hotset-8-backends-long.yaml b/src/libraries/rust/stargate/benches/hotset-8-backends-long.yaml index b0107d2f8..dc588cfb8 100644 --- a/src/libraries/rust/stargate/benches/hotset-8-backends-long.yaml +++ b/src/libraries/rust/stargate/benches/hotset-8-backends-long.yaml @@ -107,9 +107,9 @@ traffic_pattern: target_rps: 5 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin @@ -121,7 +121,7 @@ algorithms: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/hotset-8-backends.yaml b/src/libraries/rust/stargate/benches/hotset-8-backends.yaml index 00ce62793..3c1d09041 100644 --- a/src/libraries/rust/stargate/benches/hotset-8-backends.yaml +++ b/src/libraries/rust/stargate/benches/hotset-8-backends.yaml @@ -107,9 +107,9 @@ traffic_pattern: target_rps: 40 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin @@ -121,7 +121,7 @@ algorithms: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/lb-balance-bursty-4c2p-2s.yaml b/src/libraries/rust/stargate/benches/lb-balance-bursty-4c2p-2s.yaml index 3bff125d0..a7cbc5df5 100644 --- a/src/libraries/rust/stargate/benches/lb-balance-bursty-4c2p-2s.yaml +++ b/src/libraries/rust/stargate/benches/lb-balance-bursty-4c2p-2s.yaml @@ -107,9 +107,9 @@ traffic_pattern: burst_period_requests: 40 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin @@ -121,7 +121,7 @@ algorithms: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/lb-balance-hotset-8c2p-4s.yaml b/src/libraries/rust/stargate/benches/lb-balance-hotset-8c2p-4s.yaml index d84fcafba..7c28f4a43 100644 --- a/src/libraries/rust/stargate/benches/lb-balance-hotset-8c2p-4s.yaml +++ b/src/libraries/rust/stargate/benches/lb-balance-hotset-8c2p-4s.yaml @@ -109,9 +109,9 @@ traffic_pattern: target_rps: 140 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin @@ -123,7 +123,7 @@ algorithms: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-4c2p-2s.yaml b/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-4c2p-2s.yaml index 62bc9c0e4..b4fdb7d1a 100644 --- a/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-4c2p-2s.yaml +++ b/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-4c2p-2s.yaml @@ -109,9 +109,9 @@ traffic_pattern: target_rps: 45 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin @@ -123,7 +123,7 @@ algorithms: default: wait-and-widen - name: wait-and-widen-affinity config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: wait-and-widen @@ -133,7 +133,7 @@ algorithms: cache_affinity_backend_selection_count: 1 - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar @@ -142,7 +142,7 @@ algorithms: require_input_tokens: true - name: pulsar-consider-kv-free-tokens config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar @@ -151,7 +151,7 @@ algorithms: consider_kv_free_tokens: true - name: pulsar-wait-and-widen config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar-wait-and-widen @@ -160,7 +160,7 @@ algorithms: require_input_tokens: true - name: pulsar-wait-and-widen-consider-kv-free-tokens config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar-wait-and-widen diff --git a/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-pulsar-wait-and-widen-slo-4c2p-1s.yaml b/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-pulsar-wait-and-widen-slo-4c2p-1s.yaml index 7053cd3c7..447e4db95 100644 --- a/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-pulsar-wait-and-widen-slo-4c2p-1s.yaml +++ b/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-pulsar-wait-and-widen-slo-4c2p-1s.yaml @@ -115,7 +115,7 @@ algorithms: pylon_queue_admission: enabled: false config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: wait-and-widen @@ -127,7 +127,7 @@ algorithms: pylon_queue_admission: enabled: false config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar @@ -138,7 +138,7 @@ algorithms: pylon_queue_admission: enabled: false config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar-wait-and-widen diff --git a/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-smoke-2c2p-1s.yaml b/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-smoke-2c2p-1s.yaml index 49b29a640..2019da9fa 100644 --- a/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-smoke-2c2p-1s.yaml +++ b/src/libraries/rust/stargate/benches/lb-balance-prefix-reuse-smoke-2c2p-1s.yaml @@ -63,9 +63,9 @@ traffic_pattern: target_rps: 20 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin @@ -77,7 +77,7 @@ algorithms: default: wait-and-widen - name: wait-and-widen-affinity config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: wait-and-widen @@ -87,7 +87,7 @@ algorithms: cache_affinity_backend_selection_count: 1 - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar @@ -96,7 +96,7 @@ algorithms: require_input_tokens: true - name: pulsar-consider-kv-free-tokens config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar @@ -105,7 +105,7 @@ algorithms: consider_kv_free_tokens: true - name: pulsar-wait-and-widen config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar-wait-and-widen @@ -114,7 +114,7 @@ algorithms: require_input_tokens: true - name: pulsar-wait-and-widen-consider-kv-free-tokens config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar-wait-and-widen diff --git a/src/libraries/rust/stargate/benches/lb-balance-smoke-2c2p-1s.yaml b/src/libraries/rust/stargate/benches/lb-balance-smoke-2c2p-1s.yaml index 0dfaf9754..0e31c2394 100644 --- a/src/libraries/rust/stargate/benches/lb-balance-smoke-2c2p-1s.yaml +++ b/src/libraries/rust/stargate/benches/lb-balance-smoke-2c2p-1s.yaml @@ -62,9 +62,9 @@ traffic_pattern: interval_ms: 40 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin @@ -76,7 +76,7 @@ algorithms: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/mixed-size-pulsar.yaml b/src/libraries/rust/stargate/benches/mixed-size-pulsar.yaml index 10d6ab5a0..06c36fbbc 100644 --- a/src/libraries/rust/stargate/benches/mixed-size-pulsar.yaml +++ b/src/libraries/rust/stargate/benches/mixed-size-pulsar.yaml @@ -118,15 +118,15 @@ traffic_pattern: p99_cap: 2048 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: wait-and-widen config: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/overload-6-backends.yaml b/src/libraries/rust/stargate/benches/overload-6-backends.yaml index 63c6838da..6bc9f8206 100644 --- a/src/libraries/rust/stargate/benches/overload-6-backends.yaml +++ b/src/libraries/rust/stargate/benches/overload-6-backends.yaml @@ -91,9 +91,9 @@ traffic_pattern: burst_period_requests: 20 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin @@ -105,7 +105,7 @@ algorithms: default: wait-and-widen - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/stair-step-2-stargates.yaml b/src/libraries/rust/stargate/benches/stair-step-2-stargates.yaml index 99161141d..c280d752b 100644 --- a/src/libraries/rust/stargate/benches/stair-step-2-stargates.yaml +++ b/src/libraries/rust/stargate/benches/stair-step-2-stargates.yaml @@ -62,15 +62,15 @@ traffic_pattern: step_requests: 20 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/sticky-hot-prefix.yaml b/src/libraries/rust/stargate/benches/sticky-hot-prefix.yaml index fbb666546..7b3072764 100644 --- a/src/libraries/rust/stargate/benches/sticky-hot-prefix.yaml +++ b/src/libraries/rust/stargate/benches/sticky-hot-prefix.yaml @@ -64,15 +64,15 @@ traffic_pattern: target_rps: 6 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin - name: pulsar config: - default: power-of-two + default: power-of-n models: dummy-model: algorithm: pulsar diff --git a/src/libraries/rust/stargate/benches/uniform-4-backends.yaml b/src/libraries/rust/stargate/benches/uniform-4-backends.yaml index 328005d70..419fb13dc 100644 --- a/src/libraries/rust/stargate/benches/uniform-4-backends.yaml +++ b/src/libraries/rust/stargate/benches/uniform-4-backends.yaml @@ -60,9 +60,9 @@ traffic_pattern: interval_ms: 100 algorithms: - - name: power-of-two + - name: power-of-n config: - default: power-of-two + default: power-of-n - name: round-robin config: default: round-robin diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/k8s/tests.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/k8s/tests.rs index 5e2ce80b8..e48495859 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/k8s/tests.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/k8s/tests.rs @@ -73,8 +73,8 @@ fn config() -> BenchmarkConfig { }), degradation: DegradationConfig::default(), algorithms: vec![AlgorithmConfig { - name: "power-of-two".to_string(), - config: serde_json::json!({"default": "power-of-two"}), + name: "power-of-n".to_string(), + config: serde_json::json!({"default": "power-of-n"}), pylon_queue_admission: None, }], } @@ -111,7 +111,7 @@ fn render_default_test_manifest(config: &BenchmarkConfig) -> RenderedManifests { &config.algorithms[0], "sgbench-sg-power", "sgbench-be-power", - r#"{"default":"power-of-two"}"#, + r#"{"default":"power-of-n"}"#, ) } @@ -275,11 +275,11 @@ fn benchmark_run(run_dir: &Path) -> BenchmarkK8sRun { fs::write(run_dir.join(manifest), "kind: List\n").expect("manifest should write"); } BenchmarkK8sRun { - algorithm_name: "power-of-two".to_string(), + algorithm_name: "power-of-n".to_string(), manifest_path: run_dir.join("manifest.json"), run_dir: run_dir.to_path_buf(), - stargate_ns: "sgbench-sg-power-of-two".to_string(), - backends_ns: "sgbench-be-power-of-two".to_string(), + stargate_ns: "sgbench-sg-power-of-n".to_string(), + backends_ns: "sgbench-be-power-of-n".to_string(), stargate_count: 2, nodeport_host: "node.test".to_string(), stargate_http_endpoint: "http://node.test:30080".to_string(), @@ -316,8 +316,8 @@ fn prepare_benchmark_k8s_run_writes_split_manifests_and_run_info() { ) .expect("benchmark k8s run should prepare"); - assert_eq!(run.stargate_ns, "sgbench-sg-power-of-two"); - assert_eq!(run.backends_ns, "sgbench-be-power-of-two"); + assert_eq!(run.stargate_ns, "sgbench-sg-power-of-n"); + assert_eq!(run.backends_ns, "sgbench-be-power-of-n"); assert_eq!(run.stargate_http_endpoint, "http://node.example:30082"); assert_eq!( run.stargate_metrics_endpoint, @@ -352,7 +352,7 @@ fn prepare_benchmark_k8s_run_writes_split_manifests_and_run_info() { "run info should read", )) .expect("run info should parse"); - assert_eq!(run_info["algorithm_name"], "power-of-two"); + assert_eq!(run_info["algorithm_name"], "power-of-n"); assert_eq!(run_info["http_node_port"], 30082); assert_eq!( run_info["stargate_http_endpoint"], @@ -369,7 +369,7 @@ fn kubectl_runner_executes_readiness_and_maintenance_commands() { let fake = FakeKubectl::new(); let kubectl = fake.runner(); let tempdir = tempfile::tempdir().expect("tempdir should create"); - let run = benchmark_run(&tempdir.path().join("run-power-of-two")); + let run = benchmark_run(&tempdir.path().join("run-power-of-n")); kubectl.apply(&run).expect("stargate manifest should apply"); kubectl @@ -412,7 +412,7 @@ fn kubectl_runner_executes_readiness_and_maintenance_commands() { &stargate_log, &[ "status: exit status: 0", - "logs for -n sgbench-sg-power-of-two logs", + "logs for -n sgbench-sg-power-of-n logs", ], ); let backend_log = read_utf8( @@ -455,7 +455,7 @@ fi ); let kubectl = fake.runner(); let tempdir = tempfile::tempdir().expect("tempdir should create"); - let run = benchmark_run(&tempdir.path().join("run-power-of-two")); + let run = benchmark_run(&tempdir.path().join("run-power-of-n")); assert_error_contains(kubectl.apply(&run), "kubectl apply failed"); assert_error_contains(kubectl.delete(&run), "kubectl delete failed"); diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/k8s_run.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/k8s_run.rs index d34ce8e8c..a54340e91 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/k8s_run.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/k8s_run.rs @@ -672,7 +672,7 @@ mod tests { let actions = RefCell::new(Vec::new()); finalize_k8s_run_with_actions( - "power-of-two", + "power-of-n", true, false, || { @@ -694,7 +694,7 @@ mod tests { ) -> Vec<&'static str> { let actions = RefCell::new(Vec::new()); finalize_k8s_run_with_actions( - "power-of-two", + "power-of-n", run_failed, keep_resources_on_failure, || { @@ -806,7 +806,7 @@ mod tests { #[test] fn k8s_replay_artifacts_use_canonical_run_filenames() { let tempdir = tempfile::tempdir().expect("tempdir should create"); - let run_dir = tempdir.path().join("run-power-of-two"); + let run_dir = tempdir.path().join("run-power-of-n"); let artifacts = K8sRunArtifacts(&run_dir); @@ -1010,8 +1010,8 @@ traffic_pattern: output_tokens: { distribution: constant, value: 20 } arrival: { distribution: constant, interval_ms: 10 } algorithms: - - name: power-of-two - config: { default: power-of-two } + - name: power-of-n + config: { default: power-of-n } "#, ) .expect("benchmark config fixture should parse") @@ -1069,11 +1069,11 @@ algorithms: ) -> BenchmarkK8sRun { std::fs::create_dir_all(run_dir).expect("run dir should create"); BenchmarkK8sRun { - algorithm_name: "power-of-two".to_string(), + algorithm_name: "power-of-n".to_string(), manifest_path: run_dir.join("manifest.json"), run_dir: run_dir.to_path_buf(), - stargate_ns: "sgbench-sg-power-of-two".to_string(), - backends_ns: "sgbench-be-power-of-two".to_string(), + stargate_ns: "sgbench-sg-power-of-n".to_string(), + backends_ns: "sgbench-be-power-of-n".to_string(), stargate_count: 1, nodeport_host: "node.test".to_string(), stargate_http_endpoint: "http://node.test:30080".to_string(), diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/main.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/main.rs index 8b409eb2e..11546d490 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/main.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/main.rs @@ -566,7 +566,7 @@ mod tests { use clap::error::ErrorKind; const TRANSPORT_DISABLED: &str = "stargate-bench transport-bench --disable-quic-send-fairness --disable-http3-grease"; - const MATERIALIZE_ARGS: &str = "stargate-bench materialize --scenario uniform-4-backends --seed 7 --algorithm power-of-two --output-dir out"; + const MATERIALIZE_ARGS: &str = "stargate-bench materialize --scenario uniform-4-backends --seed 7 --algorithm power-of-n --output-dir out"; const PREPARE_ARGS: &str = "stargate-bench prepare-run --config config.yaml"; const RUN_ARGS: &str = "stargate-bench run --scenario uniform-4-backends --keep-resources-on-failure --reliability-mode controlled"; const STARGATE_REQUEST_METRIC: &str = concat!( @@ -612,7 +612,7 @@ mod tests { assert_eq!(conflict.kind(), ErrorKind::ArgumentConflict); assert_eq!( command_json!(MATERIALIZE_ARGS, Materialize), - r#"{"source":{"config":null,"scenario":"uniform-4-backends"},"seed":7,"algorithms":["power-of-two"],"output_dir":"out"}"# + r#"{"source":{"config":null,"scenario":"uniform-4-backends"},"seed":7,"algorithms":["power-of-n"],"output_dir":"out"}"# ); assert_eq!( command_json!(PREPARE_ARGS, PrepareRun), @@ -703,7 +703,7 @@ pylon_requests_total_total{model="dummy-model",status="complete"} 3 concurrency: 1, candidates: 2, cache_key_count: 1, - scenarios: vec![LbMicrobenchScenario::PowerOfTwo], + scenarios: vec![LbMicrobenchScenario::PowerOfN], } .execute() .expect("real command should run lb microbench"); @@ -729,7 +729,7 @@ pylon_requests_total_total{model="dummy-model",status="complete"} 3 #[test] fn run_info_reader_ignores_extra_run_metadata_fields() { let tempdir = tempfile::tempdir().expect("tempdir should create"); - let run_dir = tempdir.path().join("run-power-of-two"); + let run_dir = tempdir.path().join("run-power-of-n"); std::fs::create_dir(&run_dir).expect("run dir should create"); std::fs::write( run_dir.join("run-info.json"), @@ -830,7 +830,7 @@ pylon_requests_total_total{model="dummy-model",status="complete"} 3 BenchmarkInputArgs { source: source(None, Some("uniform-4-backends")), seed: Some(123), - algorithms: ["random", "power-of-two"].map(str::to_string).to_vec(), + algorithms: ["random", "power-of-n"].map(str::to_string).to_vec(), output_dir, } } @@ -971,7 +971,7 @@ pylon_requests_total_total{model="dummy-model",status="complete"} 3 .expect("benchmark input should load"); assert_eq!(manifest.seed, 123); - assert_eq!(algorithm_names(&config), ["power-of-two", "random"]); + assert_eq!(algorithm_names(&config), ["power-of-n", "random"]); assert_eq!(output_dir, Path::new(".bench-out").join(&config.name)); } @@ -991,11 +991,11 @@ pylon_requests_total_total{model="dummy-model",status="complete"} 3 .expect("summary should load"); assert_eq!(manifest.seed, 123); - assert_eq!(algorithm_names(&config_copy), ["power-of-two", "random"]); + assert_eq!(algorithm_names(&config_copy), ["power-of-n", "random"]); assert_eq!(summary["seed"], 123); assert_eq!( summary["algorithm_names"], - serde_json::json!(["power-of-two", "random"]) + serde_json::json!(["power-of-n", "random"]) ); } @@ -1030,7 +1030,7 @@ pylon_requests_total_total{model="dummy-model",status="complete"} 3 filter_algorithms(&mut config, &benchmark_args(None).algorithms) .expect("algorithm filter should succeed"); - assert_eq!(algorithm_names(&config), ["power-of-two", "random"]); + assert_eq!(algorithm_names(&config), ["power-of-n", "random"]); } #[test] @@ -1046,14 +1046,14 @@ pylon_requests_total_total{model="dummy-model",status="complete"} 3 #[test] fn load_balance_sweep_scenarios_cover_grouped_topologies_and_all_algorithms() { const STANDARD_ALGORITHMS: &[&str] = &[ - "power-of-two", + "power-of-n", "round-robin", "random", "wait-and-widen", "pulsar", ]; const PREFIX_ALGORITHMS: &[&str] = &[ - "power-of-two", + "power-of-n", "round-robin", "random", "wait-and-widen", diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/manifest.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/manifest.rs index 01ed8e926..258746d18 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/manifest.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/manifest.rs @@ -374,8 +374,8 @@ mod tests { }), degradation: crate::config::DegradationConfig::default(), algorithms: vec![AlgorithmConfig { - name: "power-of-two".to_string(), - config: serde_json::json!({ "default": "power-of-two" }), + name: "power-of-n".to_string(), + config: serde_json::json!({ "default": "power-of-n" }), pylon_queue_admission: None, }], } diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs index e22443f81..78a83abdc 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/microbench/lb.rs @@ -26,7 +26,7 @@ use clap::ValueEnum; use stargate::load_balancer::{ LoadBalancerAlgorithm, LoadBalancerAlgorithmConfig, LoadBalancerConfig, LoadBalancerModelConfig, LoadBalancerRequest, LoadBalancerRouter, LoadBalancerTargetState, - MAX_POWER_OF_TWO_SAMPLE_COUNT, + MAX_POWER_OF_N_SAMPLE_COUNT, }; use stargate::routing::{RoutedClusterSnapshot, RoutingTargetKey}; use stargate_proto::pb::{InferenceServerStatus, ModelStats}; @@ -68,12 +68,12 @@ macro_rules! scenarios { // Keep the scenario matrix row-oriented so differences remain directly comparable. #[rustfmt::skip] scenarios! { - PowerOfTwo, "lb-bench-power-of-two", PowerOfTwo, None, 0; - PowerOfTwoSample1, "lb-bench-power-of-two-sample-1", PowerOfTwo, None, 0; - PowerOfTwoSample4, "lb-bench-power-of-two-sample-4", PowerOfTwo, None, 0; - PowerOfTwoSample8, "lb-bench-power-of-two-sample-8", PowerOfTwo, None, 0; - PowerOfTwoFullPool, "lb-bench-power-of-two-full-pool", PowerOfTwo, None, 0; - PowerOfTwoOneExcluded, "lb-bench-power-of-two-one-excluded", PowerOfTwo, None, 1; + PowerOfN, "lb-bench-power-of-n", PowerOfN, None, 0; + PowerOfNSample1, "lb-bench-power-of-n-sample-1", PowerOfN, None, 0; + PowerOfNSample4, "lb-bench-power-of-n-sample-4", PowerOfN, None, 0; + PowerOfNSample8, "lb-bench-power-of-n-sample-8", PowerOfN, None, 0; + PowerOfNFullPool, "lb-bench-power-of-n-full-pool", PowerOfN, None, 0; + PowerOfNOneExcluded, "lb-bench-power-of-n-one-excluded", PowerOfN, None, 1; WaitAndWiden, "lb-bench-wait-and-widen", WaitAndWiden, None, 0; WaitAndWidenOneExcluded, "lb-bench-wait-and-widen-one-excluded", WaitAndWiden, None, 1; WaitAndWidenIgnoreQueue, "lb-bench-wait-and-widen-ignore-queue", WaitAndWiden, Some(IgnoreQueue), 0; @@ -100,11 +100,11 @@ impl LbMicrobenchScenario { fn configured_sample_count(self, candidate_count: usize) -> Option { match self { - Self::PowerOfTwo | Self::PowerOfTwoOneExcluded => Some(2), - Self::PowerOfTwoSample1 => Some(1), - Self::PowerOfTwoSample4 => Some(4), - Self::PowerOfTwoSample8 => Some(8), - Self::PowerOfTwoFullPool => Some(candidate_count), + Self::PowerOfN | Self::PowerOfNOneExcluded => Some(2), + Self::PowerOfNSample1 => Some(1), + Self::PowerOfNSample4 => Some(4), + Self::PowerOfNSample8 => Some(8), + Self::PowerOfNFullPool => Some(candidate_count), _ => None, } } @@ -211,14 +211,14 @@ fn validate_config(config: &LbMicrobenchConfig) -> anyhow::Result<()> { anyhow::bail!("{flag} must be greater than 0"); } } - if config.candidates > MAX_POWER_OF_TWO_SAMPLE_COUNT + if config.candidates > MAX_POWER_OF_N_SAMPLE_COUNT && (config.scenarios.is_empty() || config .scenarios - .contains(&LbMicrobenchScenario::PowerOfTwoFullPool)) + .contains(&LbMicrobenchScenario::PowerOfNFullPool)) { anyhow::bail!( - "power-of-two-full-pool requires --candidates to be at most {MAX_POWER_OF_TWO_SAMPLE_COUNT}" + "power-of-n-full-pool requires --candidates to be at most {MAX_POWER_OF_N_SAMPLE_COUNT}" ); } Ok(()) @@ -232,7 +232,7 @@ fn run_scenario( ) -> anyhow::Result { let algorithm_config = config_for_scenario(scenario, config.candidates); let router = LoadBalancerRouter::from_config(&LoadBalancerConfig { - default: LoadBalancerAlgorithm::PowerOfTwo, + default: LoadBalancerAlgorithm::PowerOfN, request_algorithms: HashMap::new(), models: HashMap::from([( scenario.metadata().model_id.to_string(), @@ -497,8 +497,8 @@ fn config_for_scenario( config.request_policy_mut().require_input_tokens = is_pulsar; if let Some(sample_count) = scenario.configured_sample_count(candidate_count) { config - .power_of_two_settings_mut() - .expect("power-of-two scenario should expose power-of-two settings") + .power_of_n_settings_mut() + .expect("power-of-n scenario should expose power-of-n settings") .sample_count = sample_count; } if is_pulsar { @@ -597,7 +597,7 @@ mod tests { #[test] fn lb_microbench_runs_default_scenarios() { - let mut config = config(LbMicrobenchScenario::PowerOfTwo); + let mut config = config(LbMicrobenchScenario::PowerOfN); config.iterations = 8; config.warmup_iterations = 2; config.candidates = 4; @@ -605,7 +605,7 @@ mod tests { let rows = run_lb_microbench(&config).expect("microbench should run"); assert_eq!(rows.len(), 23); - assert_eq!(rows[0].scenario, LbMicrobenchScenario::PowerOfTwo); + assert_eq!(rows[0].scenario, LbMicrobenchScenario::PowerOfN); for row in rows { assert_eq!(row.choices, row.iterations); assert_eq!(row.concurrency, 2); @@ -649,13 +649,13 @@ mod tests { } #[test] - fn power_of_two_microbench_scenarios_cover_requested_sample_counts() { + fn power_of_n_microbench_scenarios_cover_requested_sample_counts() { for (scenario, expected_sample_count) in [ - (LbMicrobenchScenario::PowerOfTwoSample1, 1), - (LbMicrobenchScenario::PowerOfTwo, 2), - (LbMicrobenchScenario::PowerOfTwoSample4, 4), - (LbMicrobenchScenario::PowerOfTwoSample8, 8), - (LbMicrobenchScenario::PowerOfTwoFullPool, 8), + (LbMicrobenchScenario::PowerOfNSample1, 1), + (LbMicrobenchScenario::PowerOfN, 2), + (LbMicrobenchScenario::PowerOfNSample4, 4), + (LbMicrobenchScenario::PowerOfNSample8, 8), + (LbMicrobenchScenario::PowerOfNFullPool, 8), ] { let rows = run_lb_microbench(&config(scenario)).expect("microbench should run"); assert_eq!( @@ -667,11 +667,11 @@ mod tests { #[test] fn full_pool_microbench_rejects_candidate_count_above_sample_limit() { - let mut config = config(LbMicrobenchScenario::PowerOfTwoFullPool); - config.candidates = MAX_POWER_OF_TWO_SAMPLE_COUNT + 1; + let mut config = config(LbMicrobenchScenario::PowerOfNFullPool); + config.candidates = MAX_POWER_OF_N_SAMPLE_COUNT + 1; let error = run_lb_microbench(&config).expect_err("oversized full pool should fail"); - assert!(error.to_string().contains("power-of-two-full-pool")); + assert!(error.to_string().contains("power-of-n-full-pool")); } #[test] @@ -704,7 +704,7 @@ mod tests { #[rustfmt::skip] exclusion_tests! { random_one_excluded_scenario_never_selects_excluded_backend: RandomOneExcluded => ["cluster-0000"]; - power_of_two_one_excluded_scenario_never_selects_excluded_backend: PowerOfTwoOneExcluded => ["cluster-0000"]; + power_of_n_one_excluded_scenario_never_selects_excluded_backend: PowerOfNOneExcluded => ["cluster-0000"]; wait_and_widen_one_excluded_scenario_never_selects_excluded_backend: WaitAndWidenOneExcluded => ["cluster-0000"]; wait_and_widen_affinity_one_excluded_scenario_never_selects_excluded_backend: WaitAndWidenAffinityOneExcluded => ["cluster-0000"]; wait_and_widen_affinity_multi_excluded_scenario_never_selects_excluded_backend: WaitAndWidenAffinityMultiExcluded => ["cluster-0000", "cluster-0001"]; diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/orchestrator.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/orchestrator.rs index f59462bc3..185127b72 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/orchestrator.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/orchestrator.rs @@ -361,7 +361,7 @@ traffic_pattern: output_tokens: { distribution: constant, value: 20 } arrival: { distribution: constant, interval_ms: 10 } algorithms: - - { name: power-of-two, config: { default: power-of-two } } + - { name: power-of-n, config: { default: power-of-n } } - { name: random, config: { default: random } } "#, ) @@ -429,7 +429,7 @@ algorithms: let run = prepared .algorithm_runs .iter() - .find(|run| run.algorithm_name == "power-of-two") + .find(|run| run.algorithm_name == "power-of-n") .expect("configured run should exist"); let run_info: serde_json::Value = serde_json::from_slice( &std::fs::read(&run.run_info_path).expect("run info should read"), @@ -448,7 +448,7 @@ algorithms: let compose = build_compose_spec( &config, &config.algorithms[0], - Path::new(".bench-out/prepare/run-power-of-two/lb-config.json"), + Path::new(".bench-out/prepare/run-power-of-n/lb-config.json"), STARGATE_GRPC_PORT, STARGATE_HTTP_PORT, STARGATE_METRICS_PORT, diff --git a/src/libraries/rust/stargate/crates/stargate-bench/src/report.rs b/src/libraries/rust/stargate/crates/stargate-bench/src/report.rs index 5f94d5c86..0f2c9dd96 100644 --- a/src/libraries/rust/stargate/crates/stargate-bench/src/report.rs +++ b/src/libraries/rust/stargate/crates/stargate-bench/src/report.rs @@ -678,7 +678,7 @@ mod tests { ..CacheSummary::default() }; - let report = render(&config(), "power-of-two", None, summary); + let report = render(&config(), "power-of-n", None, summary); assert!(report.contains("| Algorithm | Admission Mode | Success |")); assert!(report.contains("Successful RPS")); @@ -753,13 +753,13 @@ mod tests { }]; let successful_report = render(&config(), "round-robin", None, successful_summary); - let failed_report = render(&config(), "power-of-two", None, failed_summary); + let failed_report = render(&config(), "power-of-n", None, failed_summary); assert!(!successful_report.contains("## Failures")); assert!(failed_report.contains("## Failures")); assert!(failed_report.contains("| Algorithm | Status | Backend | Count | Error |")); assert!( - failed_report.contains("| power-of-two | 503 | backend-a | 2 | upstream unavailable |") + failed_report.contains("| power-of-n | 503 | backend-a | 2 | upstream unavailable |") ); } diff --git a/src/libraries/rust/stargate/crates/stargate/src/http_proxy/request.rs b/src/libraries/rust/stargate/crates/stargate/src/http_proxy/request.rs index eddbea1fd..fe456939f 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/http_proxy/request.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/http_proxy/request.rs @@ -279,7 +279,7 @@ mod tests { #[test] fn proxy_valid_configured_routing_method_uses_request_algorithm() { let lb_router = LoadBalancerRouter::from_config(&LoadBalancerConfig { - default: LoadBalancerAlgorithm::PowerOfTwo, + default: LoadBalancerAlgorithm::PowerOfN, request_algorithms: HashMap::from([( LoadBalancerAlgorithm::RoundRobin, LoadBalancerModelConfig::Name(LoadBalancerAlgorithm::RoundRobin), diff --git a/src/libraries/rust/stargate/crates/stargate/src/http_proxy/routing.rs b/src/libraries/rust/stargate/crates/stargate/src/http_proxy/routing.rs index cd4b49c86..e5c4ad3ef 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/http_proxy/routing.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/http_proxy/routing.rs @@ -286,7 +286,7 @@ mod tests { let mut candidate = cluster_candidate("cluster-a"); candidate.stats.queued_input_size = 300; candidate.stats.last_mean_input_tps = 100.0; - let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfTwo); + let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfN); let target = routing_target(); let request = input_work_admission_request(&target, 50); @@ -302,7 +302,7 @@ mod tests { candidate.stats.total_query_input_size = 300; candidate.stats.queued_input_size = 0; candidate.stats.last_mean_input_tps = 100.0; - let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfTwo); + let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfN); let target = routing_target(); let request = input_work_admission_request(&target, 50); @@ -316,7 +316,7 @@ mod tests { fn input_work_admission_rejects_pool_without_valid_capacity() { let mut candidate = cluster_candidate("cluster-a"); candidate.stats.last_mean_input_tps = 0.0; - let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfTwo); + let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfN); let target = routing_target(); let request = input_work_admission_request(&target, 50); diff --git a/src/libraries/rust/stargate/crates/stargate/src/http_proxy/run.rs b/src/libraries/rust/stargate/crates/stargate/src/http_proxy/run.rs index 9cb040cf6..ee02dcecd 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/http_proxy/run.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/http_proxy/run.rs @@ -465,7 +465,7 @@ mod tests { rank_depth: 1, selected_after_kv_free_tokens_skip: false, }, - effective_algorithm: LoadBalancerAlgorithm::PowerOfTwo, + effective_algorithm: LoadBalancerAlgorithm::PowerOfN, requested_algorithm: None, }; let selected_cluster = SelectedClusterRun::new( @@ -479,7 +479,7 @@ mod tests { let body = metrics_text(&app.metrics); assert!( body.contains( - r#"stargate_routing_selections_total{algorithm="power-of-two",model="model-a",routing_key="tenant-a",selection="primary"} 1"# + r#"stargate_routing_selections_total{algorithm="power-of-n",model="model-a",routing_key="tenant-a",selection="primary"} 1"# ), "selected cluster should preserve routing selection metric labels, got:\n{body}" ); diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs index e26618e9c..6c4c33260 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs @@ -49,7 +49,14 @@ impl<'de> Deserialize<'de> for LoadBalancerModelConfig { #[serde(rename_all = "kebab-case")] pub enum LoadBalancerAlgorithm { #[default] - PowerOfTwo, + #[serde( + alias = "power-of-two", + alias = "powerOf2", + alias = "powerOfN", + alias = "powerof2", + alias = "powerofn" + )] + PowerOfN, #[serde(alias = "groq-multiregion")] WaitAndWiden, RoundRobin, @@ -61,7 +68,7 @@ pub enum LoadBalancerAlgorithm { impl LoadBalancerAlgorithm { pub const ALL: [Self; 6] = [ - Self::PowerOfTwo, + Self::PowerOfN, Self::WaitAndWiden, Self::RoundRobin, Self::Random, @@ -73,7 +80,7 @@ impl LoadBalancerAlgorithm { impl fmt::Display for LoadBalancerAlgorithm { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let name = match self { - Self::PowerOfTwo => "power-of-two", + Self::PowerOfN => "power-of-n", Self::WaitAndWiden => "wait-and-widen", Self::RoundRobin => "round-robin", Self::Random => "random", @@ -194,29 +201,29 @@ pub struct WaitAndWidenAlgorithmConfig { pub ignore_input_processing_time: Option, } -const DEFAULT_POWER_OF_TWO_SAMPLE_COUNT: usize = 2; -pub const MAX_POWER_OF_TWO_SAMPLE_COUNT: usize = 64; +const DEFAULT_POWER_OF_N_SAMPLE_COUNT: usize = 2; +pub const MAX_POWER_OF_N_SAMPLE_COUNT: usize = 64; #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(default)] -pub struct PowerOfTwoAlgorithmConfig { +pub struct PowerOfNAlgorithmConfig { pub sample_count: usize, } -impl Default for PowerOfTwoAlgorithmConfig { +impl Default for PowerOfNAlgorithmConfig { fn default() -> Self { Self { - sample_count: DEFAULT_POWER_OF_TWO_SAMPLE_COUNT, + sample_count: DEFAULT_POWER_OF_N_SAMPLE_COUNT, } } } -impl PowerOfTwoAlgorithmConfig { +impl PowerOfNAlgorithmConfig { pub(crate) fn validated_sample_count(&self) -> Result { let sample_count = self.sample_count; - if !(1..=MAX_POWER_OF_TWO_SAMPLE_COUNT).contains(&sample_count) { + if !(1..=MAX_POWER_OF_N_SAMPLE_COUNT).contains(&sample_count) { return Err(format!( - "power-of-two sample_count must be between 1 and {MAX_POWER_OF_TWO_SAMPLE_COUNT}, got {sample_count}" + "power-of-n sample_count must be between 1 and {MAX_POWER_OF_N_SAMPLE_COUNT}, got {sample_count}" )); } Ok(sample_count) @@ -225,7 +232,7 @@ impl PowerOfTwoAlgorithmConfig { #[derive(Debug, Clone, PartialEq)] pub enum LoadBalancerAlgorithmSettings { - PowerOfTwo(PowerOfTwoAlgorithmConfig), + PowerOfN(PowerOfNAlgorithmConfig), WaitAndWiden(WaitAndWidenAlgorithmConfig), RoundRobin, Random, @@ -235,14 +242,14 @@ pub enum LoadBalancerAlgorithmSettings { impl Default for LoadBalancerAlgorithmSettings { fn default() -> Self { - Self::PowerOfTwo(PowerOfTwoAlgorithmConfig::default()) + Self::PowerOfN(PowerOfNAlgorithmConfig::default()) } } impl LoadBalancerAlgorithmSettings { fn algorithm(&self) -> LoadBalancerAlgorithm { match self { - Self::PowerOfTwo(_) => LoadBalancerAlgorithm::PowerOfTwo, + Self::PowerOfN(_) => LoadBalancerAlgorithm::PowerOfN, Self::WaitAndWiden(_) => LoadBalancerAlgorithm::WaitAndWiden, Self::RoundRobin => LoadBalancerAlgorithm::RoundRobin, Self::Random => LoadBalancerAlgorithm::Random, @@ -286,7 +293,7 @@ impl LoadBalancerAlgorithmConfig { LoadBalancerAlgorithmSettings::Pulsar(seed) => seed.as_deref(), LoadBalancerAlgorithmSettings::WaitAndWiden(config) | LoadBalancerAlgorithmSettings::PulsarWaitAndWiden(config) => config.seed.as_deref(), - LoadBalancerAlgorithmSettings::PowerOfTwo(_) + LoadBalancerAlgorithmSettings::PowerOfN(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random => None, } @@ -308,7 +315,7 @@ impl LoadBalancerAlgorithmConfig { config.seed = seed; Ok(()) } - LoadBalancerAlgorithmSettings::PowerOfTwo(_) + LoadBalancerAlgorithmSettings::PowerOfN(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random => { Err(LoadBalancerSeedError::Unsupported { algorithm }) @@ -320,7 +327,7 @@ impl LoadBalancerAlgorithmConfig { match &self.settings { LoadBalancerAlgorithmSettings::WaitAndWiden(config) | LoadBalancerAlgorithmSettings::PulsarWaitAndWiden(config) => Some(config), - LoadBalancerAlgorithmSettings::PowerOfTwo(_) + LoadBalancerAlgorithmSettings::PowerOfN(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random | LoadBalancerAlgorithmSettings::Pulsar(_) => None, @@ -331,16 +338,16 @@ impl LoadBalancerAlgorithmConfig { match &mut self.settings { LoadBalancerAlgorithmSettings::WaitAndWiden(config) | LoadBalancerAlgorithmSettings::PulsarWaitAndWiden(config) => Some(config), - LoadBalancerAlgorithmSettings::PowerOfTwo(_) + LoadBalancerAlgorithmSettings::PowerOfN(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random | LoadBalancerAlgorithmSettings::Pulsar(_) => None, } } - pub fn power_of_two_settings(&self) -> Option<&PowerOfTwoAlgorithmConfig> { + pub fn power_of_n_settings(&self) -> Option<&PowerOfNAlgorithmConfig> { match &self.settings { - LoadBalancerAlgorithmSettings::PowerOfTwo(config) => Some(config), + LoadBalancerAlgorithmSettings::PowerOfN(config) => Some(config), LoadBalancerAlgorithmSettings::WaitAndWiden(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random @@ -349,9 +356,9 @@ impl LoadBalancerAlgorithmConfig { } } - pub fn power_of_two_settings_mut(&mut self) -> Option<&mut PowerOfTwoAlgorithmConfig> { + pub fn power_of_n_settings_mut(&mut self) -> Option<&mut PowerOfNAlgorithmConfig> { match &mut self.settings { - LoadBalancerAlgorithmSettings::PowerOfTwo(config) => Some(config), + LoadBalancerAlgorithmSettings::PowerOfN(config) => Some(config), LoadBalancerAlgorithmSettings::WaitAndWiden(_) | LoadBalancerAlgorithmSettings::RoundRobin | LoadBalancerAlgorithmSettings::Random @@ -373,7 +380,7 @@ impl From for LoadBalancerAlgorithmConfig { impl From for LoadBalancerAlgorithmSettings { fn from(algorithm: LoadBalancerAlgorithm) -> Self { match algorithm { - LoadBalancerAlgorithm::PowerOfTwo => Self::PowerOfTwo(Default::default()), + LoadBalancerAlgorithm::PowerOfN => Self::PowerOfN(Default::default()), LoadBalancerAlgorithm::WaitAndWiden => Self::WaitAndWiden(Default::default()), LoadBalancerAlgorithm::RoundRobin => Self::RoundRobin, LoadBalancerAlgorithm::Random => Self::Random, @@ -430,9 +437,10 @@ impl RawCommonAlgorithmConfig { #[derive(Debug, Deserialize)] #[serde(tag = "algorithm", rename_all = "kebab-case")] enum RawLoadBalancerAlgorithmConfig { - PowerOfTwo { + #[serde(alias = "power-of-two", alias = "powerOf2", alias = "powerOfN")] + PowerOfN { #[serde(flatten)] - settings: PowerOfTwoAlgorithmConfig, + settings: PowerOfNAlgorithmConfig, #[serde(flatten)] common: RawCommonAlgorithmConfig, }, @@ -470,9 +478,9 @@ impl RawLoadBalancerAlgorithmConfig { Option, ) { match self { - Self::PowerOfTwo { settings, common } => ( + Self::PowerOfN { settings, common } => ( common, - LoadBalancerAlgorithmSettings::PowerOfTwo(settings), + LoadBalancerAlgorithmSettings::PowerOfN(settings), None, ), Self::WaitAndWiden { settings, common } => ( @@ -505,7 +513,7 @@ impl RawLoadBalancerAlgorithmConfig { fn into_config(self) -> Result { let (common, settings, consider_kv_free_tokens) = self.normalized(); - if let LoadBalancerAlgorithmSettings::PowerOfTwo(config) = &settings { + if let LoadBalancerAlgorithmSettings::PowerOfN(config) = &settings { config.validated_sample_count()?; } common.into_config(settings, consider_kv_free_tokens) @@ -544,7 +552,7 @@ pub struct LoadBalancerConfig { } impl LoadBalancerConfig { - /// Config used when no config file is given: `power-of-two` by default, + /// Config used when no config file is given: `power-of-n` by default, /// and any built-in algorithm can be selected per request. pub fn permissive_default() -> Self { Self { diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/factory.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/factory.rs index 321a02cb2..197fe4764 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/factory.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/factory.rs @@ -15,7 +15,7 @@ use std::sync::Arc; -use super::power_of_two::PowerOfTwoLoadBalancer; +use super::power_of_n::PowerOfNLoadBalancer; use super::pulsar::PulsarLoadBalancer; use super::pulsar_wait_and_widen::PulsarWaitAndWidenLoadBalancer; use super::random::RandomLoadBalancer; @@ -38,8 +38,8 @@ pub fn create_load_balancer_with_config( } match config.algorithm() { - LoadBalancerAlgorithm::PowerOfTwo => Ok(Arc::new( - PowerOfTwoLoadBalancer::from_algorithm_config(config)?, + LoadBalancerAlgorithm::PowerOfN => Ok(Arc::new( + PowerOfNLoadBalancer::from_algorithm_config(config)?, )), LoadBalancerAlgorithm::WaitAndWiden => Ok(Arc::new(WaitAndWidenLoadBalancer::new( WaitAndWidenConfig::from_algorithm_config(config), diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs index 7618ffe77..0259ab147 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/mod.rs @@ -26,7 +26,7 @@ macro_rules! impl_display { mod algorithm; mod config; mod factory; -mod power_of_two; +mod power_of_n; mod pulsar; mod pulsar_wait_and_widen; mod random; @@ -45,7 +45,7 @@ pub use config::{ LoadBalancerAlgorithm, LoadBalancerAlgorithmConfig, LoadBalancerAlgorithmOverride, LoadBalancerAlgorithmSettings, LoadBalancerConfig, LoadBalancerModelConfig, LoadBalancerRequestPolicy, LoadBalancerRoutingAlgorithmError, LoadBalancerSeedError, - MAX_POWER_OF_TWO_SAMPLE_COUNT, PowerOfTwoAlgorithmConfig, WaitAndWidenAlgorithmConfig, + MAX_POWER_OF_N_SAMPLE_COUNT, PowerOfNAlgorithmConfig, WaitAndWidenAlgorithmConfig, }; pub use factory::create_load_balancer_with_config; pub use request::{LoadBalancerCandidateChoice, LoadBalancerRequest}; diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_two.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_n.rs similarity index 95% rename from src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_two.rs rename to src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_n.rs index 14504842b..4ee8cc6b0 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_two.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/power_of_n.rs @@ -22,21 +22,21 @@ use tracing::{Span, debug}; use super::tests::LoadBalancerTestChoiceExt; use super::{ LoadBalancer, LoadBalancerAlgorithmConfig, LoadBalancerCandidateChoice, LoadBalancerRequest, - MAX_POWER_OF_TWO_SAMPLE_COUNT, + MAX_POWER_OF_N_SAMPLE_COUNT, }; use crate::routing_state::RoutedClusterSnapshot; -pub(super) struct PowerOfTwoLoadBalancer { +pub(super) struct PowerOfNLoadBalancer { sample_count: usize, } -impl PowerOfTwoLoadBalancer { +impl PowerOfNLoadBalancer { pub(super) fn from_algorithm_config( config: &LoadBalancerAlgorithmConfig, ) -> anyhow::Result { let settings = config - .power_of_two_settings() - .expect("power-of-two settings should match load-balancer algorithm"); + .power_of_n_settings() + .expect("power-of-n settings should match load-balancer algorithm"); let sample_count = settings .validated_sample_count() .map_err(anyhow::Error::msg)?; @@ -58,9 +58,9 @@ impl PowerOfTwoLoadBalancer { } } -impl_display!(PowerOfTwoLoadBalancer, "power-of-two"); +impl_display!(PowerOfNLoadBalancer, "power-of-n"); -impl LoadBalancer for PowerOfTwoLoadBalancer { +impl LoadBalancer for PowerOfNLoadBalancer { fn choose_candidate( &self, request: &LoadBalancerRequest<'_>, @@ -72,14 +72,14 @@ impl LoadBalancer for PowerOfTwoLoadBalancer { } struct CandidateSample { - indices: [usize; MAX_POWER_OF_TWO_SAMPLE_COUNT], + indices: [usize; MAX_POWER_OF_N_SAMPLE_COUNT], len: usize, } impl CandidateSample { fn new() -> Self { Self { - indices: [0; MAX_POWER_OF_TWO_SAMPLE_COUNT], + indices: [0; MAX_POWER_OF_N_SAMPLE_COUNT], len: 0, } } @@ -236,7 +236,7 @@ mod tests { request_slo: None, excluded_cluster_ids: Some(excluded_cluster_ids), }; - PowerOfTwoLoadBalancer { sample_count } + PowerOfNLoadBalancer { sample_count } .choose_for_test(&request, candidates) .map(|choice| choice.candidate.cluster_id) } @@ -286,7 +286,7 @@ mod tests { } #[test] - fn power_of_two_never_selects_excluded_clusters() { + fn power_of_n_never_selects_excluded_clusters() { let candidates = vec![ candidate("excluded-a", 1_000.0, 0), candidate("eligible", 1.0, 0), @@ -303,7 +303,7 @@ mod tests { } #[test] - fn power_of_two_skips_single_excluded_cluster_in_retry_set() { + fn power_of_n_skips_single_excluded_cluster_in_retry_set() { let candidates = (0..64) .map(|index| candidate(&format!("cluster-{index:04}"), 1_000.0, 0)) .collect::>(); @@ -317,7 +317,7 @@ mod tests { } #[test] - fn power_of_two_returns_none_when_all_candidates_are_excluded() { + fn power_of_n_returns_none_when_all_candidates_are_excluded() { let candidates = vec![ candidate("excluded-a", 1_000.0, 0), candidate("excluded-b", 1_000.0, 0), @@ -450,7 +450,7 @@ mod tests { request_slo: None, excluded_cluster_ids: None, }; - let load_balancer = PowerOfTwoLoadBalancer { sample_count: 3 }; + let load_balancer = PowerOfNLoadBalancer { sample_count: 3 }; let mut rng = StdRng::seed_from_u64(11); let mut selected = HashSet::new(); diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs index 64b413720..5009697fb 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs @@ -144,7 +144,7 @@ fn seeded_pulsar_algorithm_config(seed: &str) -> LoadBalancerAlgorithmConfig { #[test] fn set_seed_reports_unsupported_algorithms_without_panicking() { for algorithm in [ - LoadBalancerAlgorithm::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfN, LoadBalancerAlgorithm::RoundRobin, LoadBalancerAlgorithm::Random, ] { @@ -571,7 +571,7 @@ where fn assert_algorithm_overrides(raw: impl Fn(LoadBalancerAlgorithm) -> String) { for algorithm in [ LoadBalancerAlgorithm::WaitAndWiden, - LoadBalancerAlgorithm::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfN, LoadBalancerAlgorithm::Pulsar, LoadBalancerAlgorithm::PulsarWaitAndWiden, LoadBalancerAlgorithm::Random, @@ -604,7 +604,7 @@ fn simple_model_config_parses_to_algorithm_enum() { #[test] fn detailed_model_config_parses_input_work_admission_limit() { let config: LoadBalancerConfig = parse_json( - r#"{"models":{"model-a":{"algorithm":"power-of-two","max_input_work_seconds":2.5}}}"#, + r#"{"models":{"model-a":{"algorithm":"power-of-n","max_input_work_seconds":2.5}}}"#, ); let detailed = config @@ -721,41 +721,41 @@ fn algorithm_specific_load_balancer_fields_are_rejected_for_other_algorithms() { } #[test] -fn power_of_two_sample_count_defaults_to_two() { - let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfTwo); +fn power_of_n_sample_count_defaults_to_two() { + let config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfN); let settings = config - .power_of_two_settings() - .expect("power-of-two config should expose settings"); + .power_of_n_settings() + .expect("power-of-n config should expose settings"); assert_eq!(settings.sample_count, 2); } #[test] -fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { +fn detailed_power_of_n_sample_count_parses_in_every_supported_context() { let direct: LoadBalancerAlgorithmConfig = - parse_json(r#"{"algorithm":"power-of-two","sample_count":1}"#); + parse_json(r#"{"algorithm":"power-of-n","sample_count":1}"#); assert_eq!( direct - .power_of_two_settings() + .power_of_n_settings() .expect("direct config should expose settings") .sample_count, 1 ); let router = router_from_json( - r#"{"default":"random","request_algorithms":{"power-of-two":{"algorithm":"power-of-two","sample_count":4}},"models":{"model-a":{"algorithm":"power-of-two","sample_count":8,"request_algorithms":{"power-of-two":{"algorithm":"power-of-two","sample_count":64}}}}}"#, + r#"{"default":"random","request_algorithms":{"power-of-n":{"algorithm":"power-of-n","sample_count":4}},"models":{"model-a":{"algorithm":"power-of-n","sample_count":8,"request_algorithms":{"power-of-n":{"algorithm":"power-of-n","sample_count":64}}}}}"#, ); assert_eq!( router .algorithm_config("model-a") - .power_of_two_settings() + .power_of_n_settings() .expect("model config should expose settings") .sample_count, 8 ); - let override_header = LoadBalancerAlgorithmOverride::parse("power-of-two") - .expect("power-of-two override should parse"); + let override_header = LoadBalancerAlgorithmOverride::parse("power-of-n") + .expect("power-of-n override should parse"); let model_override = router .resolve_algorithm_override("model-a", Some(&override_header)) .expect("model override should resolve"); @@ -765,7 +765,7 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { assert_eq!( model_override .config() - .power_of_two_settings() + .power_of_n_settings() .expect("model override should expose settings") .sample_count, 8, @@ -774,14 +774,14 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { assert_eq!( default_override .config() - .power_of_two_settings() + .power_of_n_settings() .expect("top-level override should expose settings") .sample_count, 4 ); let nested_router = router_from_json( - r#"{"default":"random","models":{"model-a":{"algorithm":"random","request_algorithms":{"power-of-two":{"algorithm":"power-of-two","sample_count":64}}}}}"#, + r#"{"default":"random","models":{"model-a":{"algorithm":"random","request_algorithms":{"power-of-n":{"algorithm":"power-of-n","sample_count":64}}}}}"#, ); let nested_override = nested_router .resolve_algorithm_override("model-a", Some(&override_header)) @@ -789,7 +789,7 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { assert_eq!( nested_override .config() - .power_of_two_settings() + .power_of_n_settings() .expect("nested override should expose settings") .sample_count, 64 @@ -797,24 +797,24 @@ fn detailed_power_of_two_sample_count_parses_in_every_supported_context() { } #[test] -fn invalid_power_of_two_sample_counts_are_rejected_with_field_context() { - for sample_count in [0, MAX_POWER_OF_TWO_SAMPLE_COUNT + 1] { +fn invalid_power_of_n_sample_counts_are_rejected_with_field_context() { + for sample_count in [0, MAX_POWER_OF_N_SAMPLE_COUNT + 1] { assert_json_rejected::( - &format!(r#"{{"algorithm":"power-of-two","sample_count":{sample_count}}}"#), - "power-of-two sample_count must be between 1 and 64", + &format!(r#"{{"algorithm":"power-of-n","sample_count":{sample_count}}}"#), + "power-of-n sample_count must be between 1 and 64", ); } - let mut config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfTwo); + let mut config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::PowerOfN); config - .power_of_two_settings_mut() - .expect("power-of-two config should expose mutable settings") + .power_of_n_settings_mut() + .expect("power-of-n config should expose mutable settings") .sample_count = 0; let error = match create_load_balancer_with_config(&config) { Ok(_) => panic!("programmatic invalid sample count should fail"), Err(error) => error, }; - assert!(error.to_string().contains("power-of-two sample_count")); + assert!(error.to_string().contains("power-of-n sample_count")); } #[test] @@ -822,7 +822,7 @@ fn detailed_algorithm_configs_preserve_all_variant_identities() { use LoadBalancerAlgorithm::*; for (raw, expected, expected_seed, considers_kv_free_tokens) in [ - (r#"{"algorithm":"power-of-two"}"#, PowerOfTwo, None, false), + (r#"{"algorithm":"power-of-n"}"#, PowerOfN, None, false), ( r#"{"algorithm":"wait-and-widen","seed":"wait-and-widen-seed"}"#, WaitAndWiden, @@ -854,7 +854,7 @@ fn detailed_algorithm_configs_preserve_all_variant_identities() { #[test] fn unknown_load_balancer_config_fields_are_rejected() { assert_json_rejected::( - r#"{"default":"power-of-two","unused_top_level_field":true,"models":{"model-a":{"algorithm":"pulsar","unused_model_field":123}}}"#, + r#"{"default":"power-of-n","unused_top_level_field":true,"models":{"model-a":{"algorithm":"pulsar","unused_model_field":123}}}"#, "unused_top_level_field", ); } @@ -926,7 +926,7 @@ fn published_load_balancer_configuration_examples_parse() { #[test] fn detailed_model_config_parses_for_pulsar() { let router = router_from_json( - r#"{"default":"power-of-two","models":{"model-a":{"algorithm":"pulsar","seed":"seed-1","require_cache_affinity_key":true,"consider_kv_free_tokens":true}}}"#, + r#"{"default":"power-of-n","models":{"model-a":{"algorithm":"pulsar","seed":"seed-1","require_cache_affinity_key":true,"consider_kv_free_tokens":true}}}"#, ); let model_config = router.algorithm_config("model-a"); assert_eq!(model_config.algorithm(), LoadBalancerAlgorithm::Pulsar); @@ -947,7 +947,7 @@ fn kv_free_token_consideration_is_rejected_for_non_pulsar_algorithms() { #[test] fn detailed_model_config_parses_for_pulsar_wait_and_widen() { let router = router_from_json( - r#"{"default":"power-of-two","models":{"model-a":{"algorithm":"pulsar-wait-and-widen","seed":"seed-1","require_cache_affinity_key":true,"require_input_tokens":true,"max_queue_time_floor_ms":100,"max_queue_time_ceil_ms":100,"ttft_bucket_size_ms":50,"n":2}}}"#, + r#"{"default":"power-of-n","models":{"model-a":{"algorithm":"pulsar-wait-and-widen","seed":"seed-1","require_cache_affinity_key":true,"require_input_tokens":true,"max_queue_time_floor_ms":100,"max_queue_time_ceil_ms":100,"ttft_bucket_size_ms":50,"n":2}}}"#, ); let model_config = router.algorithm_config("model-a"); assert_eq!( @@ -1002,10 +1002,35 @@ fn legacy_algorithm_names_remain_compatible_in_short_configs() { } } +#[test] +fn power_of_n_aliases_remain_compatible() { + for alias in ["power-of-two", "powerOf2", "powerOfN"] { + let config: LoadBalancerConfig = parse_json(&format!(r#"{{"default":"{alias}"}}"#)); + assert_eq!(config.default, LoadBalancerAlgorithm::PowerOfN, "{alias}"); + + let detailed: LoadBalancerAlgorithmConfig = + parse_json(&format!(r#"{{"algorithm":"{alias}"}}"#)); + assert_eq!( + detailed.algorithm(), + LoadBalancerAlgorithm::PowerOfN, + "{alias}" + ); + + let algorithm_override = LoadBalancerAlgorithmOverride::parse(alias) + .expect("power-of-n alias should remain compatible"); + assert_eq!( + algorithm_override.algorithm(), + LoadBalancerAlgorithm::PowerOfN, + "{alias}" + ); + assert_eq!(algorithm_override.requested_algorithm(), alias); + } +} + #[test] fn legacy_algorithm_names_remain_compatible_in_detailed_configs() { let router = router_from_json( - r#"{"default":"power-of-two","models":{"wait-model":{"algorithm":"groq-multiregion","seed":"seed-1"},"pulsar-model":{"algorithm":"pulsar-multiregion","seed":"seed-2","max_queue_time_floor_ms":100,"max_queue_time_ceil_ms":200}}}"#, + r#"{"default":"power-of-n","models":{"wait-model":{"algorithm":"groq-multiregion","seed":"seed-1"},"pulsar-model":{"algorithm":"pulsar-multiregion","seed":"seed-2","max_queue_time_floor_ms":100,"max_queue_time_ceil_ms":200}}}"#, ); assert_eq!( @@ -1025,7 +1050,7 @@ fn legacy_algorithm_names_remain_compatible_in_detailed_configs() { #[test] fn detailed_model_config_parses_wait_and_widen_cache_affinity() { let router = router_from_json( - r#"{"default":"power-of-two","models":{"model-a":{"algorithm":"wait-and-widen","seed":"seed-1","require_cache_affinity_key":true,"cache_affinity_virtual_nodes":64,"cache_affinity_backend_selection_count":2}}}"#, + r#"{"default":"power-of-n","models":{"model-a":{"algorithm":"wait-and-widen","seed":"seed-1","require_cache_affinity_key":true,"cache_affinity_virtual_nodes":64,"cache_affinity_backend_selection_count":2}}}"#, ); let model_config = router.algorithm_config("model-a"); assert_eq!( @@ -1045,7 +1070,7 @@ fn detailed_model_config_parses_wait_and_widen_cache_affinity() { #[test] fn request_algorithms_parse_and_override_default_selection() { let router = router_from_json( - r#"{"default":"power-of-two","request_algorithms":{"round-robin":"round-robin"}}"#, + r#"{"default":"power-of-n","request_algorithms":{"round-robin":"round-robin"}}"#, ); let target = target_with_model("model-a"); let request = request(&target, None, None); @@ -1085,7 +1110,7 @@ fn choose_candidate_returns_slice_index_for_selected_cluster() { #[test] fn choose_candidate_with_resolution_preserves_algorithm_metadata() { let router = router_from_json( - r#"{"default":"power-of-two","request_algorithms":{"round-robin":"round-robin"}}"#, + r#"{"default":"power-of-n","request_algorithms":{"round-robin":"round-robin"}}"#, ); let target = target_with_model("model-a"); let request = request(&target, None, None); @@ -1120,7 +1145,7 @@ fn choose_candidate_with_resolution_preserves_algorithm_metadata() { #[test] fn model_request_algorithms_override_top_level_request_algorithms() { let router = router_from_json( - r#"{"default":"power-of-two","request_algorithms":{"round-robin":"round-robin"},"models":{"model-a":{"algorithm":"power-of-two","request_algorithms":{"round-robin":{"algorithm":"round-robin","require_input_tokens":true}}}}}"#, + r#"{"default":"power-of-n","request_algorithms":{"round-robin":"round-robin"},"models":{"model-a":{"algorithm":"power-of-n","request_algorithms":{"round-robin":{"algorithm":"round-robin","require_input_tokens":true}}}}}"#, ); let algorithm_override = LoadBalancerAlgorithmOverride::parse("round-robin") .expect("routing algorithm override should parse"); @@ -1139,7 +1164,7 @@ fn model_request_algorithms_override_top_level_request_algorithms() { #[test] fn request_algorithm_key_must_match_configured_algorithm() { let config: LoadBalancerConfig = - parse_json(r#"{"default":"power-of-two","request_algorithms":{"random":"round-robin"}}"#); + parse_json(r#"{"default":"power-of-n","request_algorithms":{"random":"round-robin"}}"#); let err = match LoadBalancerRouter::from_config(&config) { Ok(_) => panic!("mismatched request algorithm should fail"), @@ -1189,7 +1214,7 @@ fn wait_and_widen_config_resolves_internal_defaults() { #[test] fn router_reports_wait_and_widen_algorithm_name() { let router = router_with_model( - LoadBalancerAlgorithm::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfN, "model-a", LoadBalancerModelConfig::Name(LoadBalancerAlgorithm::WaitAndWiden), ); @@ -1268,7 +1293,7 @@ fn target_state_distinguishes_independent_router_definitions() { #[test] fn configured_round_robin_uses_independent_sequences_per_routing_target() { let router = router_with_model( - LoadBalancerAlgorithm::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfN, "shared-model", LoadBalancerModelConfig::Name(LoadBalancerAlgorithm::RoundRobin), ); @@ -1301,7 +1326,7 @@ fn choose_with_no_candidates_does_not_cache_default_lb_for_target() { #[test] fn request_round_robin_override_uses_stable_per_target_sequence() { let router = router_with_options( - LoadBalancerAlgorithm::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfN, &[LoadBalancerAlgorithm::RoundRobin], None, ); @@ -1330,24 +1355,19 @@ fn request_round_robin_override_uses_stable_per_target_sequence() { fn configured_request_override_creates_target_local_balancer() { let router = router_with_options( LoadBalancerAlgorithm::RoundRobin, - &[LoadBalancerAlgorithm::PowerOfTwo], + &[LoadBalancerAlgorithm::PowerOfN], None, ); let target = target_with_model("model-a"); let request = request(&target, None, None); let candidates = candidates(&["cluster-0", "cluster-1"]); let target_state = LoadBalancerTargetState::default(); - let selection = choose_with_override( - &router, - &target_state, - &request, - &candidates, - "power-of-two", - ); + let selection = + choose_with_override(&router, &target_state, &request, &candidates, "power-of-n"); assert_eq!( selection.effective_algorithm, - LoadBalancerAlgorithm::PowerOfTwo + LoadBalancerAlgorithm::PowerOfN ); assert_eq!(target_state.instance_count(), 1); } @@ -1375,7 +1395,7 @@ fn matching_round_robin_override_reuses_configured_target_sequence() { #[test] fn request_round_robin_override_keeps_routing_targets_isolated() { let router = router_with_options( - LoadBalancerAlgorithm::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfN, &[LoadBalancerAlgorithm::RoundRobin], None, ); @@ -1400,8 +1420,8 @@ fn request_round_robin_override_keeps_routing_targets_isolated() { #[test] fn request_override_beats_configured_model_algorithm() { let router = router_with_options( - LoadBalancerAlgorithm::PowerOfTwo, - &[LoadBalancerAlgorithm::PowerOfTwo], + LoadBalancerAlgorithm::PowerOfN, + &[LoadBalancerAlgorithm::PowerOfN], Some(( "shared-model", LoadBalancerModelConfig::Name(LoadBalancerAlgorithm::RoundRobin), @@ -1411,17 +1431,12 @@ fn request_override_beats_configured_model_algorithm() { let request = request(&target, None, None); let candidates = candidates(&["cluster-0", "cluster-1"]); let target_state = LoadBalancerTargetState::default(); - let selection = choose_with_override( - &router, - &target_state, - &request, - &candidates, - "power_of_two", - ); + let selection = + choose_with_override(&router, &target_state, &request, &candidates, "power_of_n"); assert_eq!( selection.effective_algorithm, - LoadBalancerAlgorithm::PowerOfTwo + LoadBalancerAlgorithm::PowerOfN ); } @@ -1431,7 +1446,7 @@ fn matching_request_override_reuses_configured_algorithm_config() { LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::RoundRobin); round_robin_config.request_policy_mut().require_input_tokens = true; let router = router_with_model( - LoadBalancerAlgorithm::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfN, "shared-model", LoadBalancerModelConfig::Detailed(Box::new(round_robin_config)), ); @@ -1454,7 +1469,7 @@ fn matching_model_algorithm_beats_top_level_request_config() { let mut pulsar_config = LoadBalancerAlgorithmConfig::from(LoadBalancerAlgorithm::Pulsar); pulsar_config.request_policy_mut().require_input_tokens = true; let router = router_with_options( - LoadBalancerAlgorithm::PowerOfTwo, + LoadBalancerAlgorithm::PowerOfN, &[LoadBalancerAlgorithm::Pulsar], Some(( "shared-model", @@ -1474,7 +1489,7 @@ fn matching_model_algorithm_beats_top_level_request_config() { #[test] fn known_unavailable_request_override_returns_error() { - let router = router_with_default(LoadBalancerAlgorithm::PowerOfTwo); + let router = router_with_default(LoadBalancerAlgorithm::PowerOfN); let target = target_with_model("shared-model"); let request = request(&target, None, None); let candidates = candidates(&["cluster-0", "cluster-1"]); @@ -1535,7 +1550,8 @@ fn permissive_default_resolves_alias_and_underscore_spellings() { let router = LoadBalancerRouter::from_config(&LoadBalancerConfig::permissive_default()) .expect("permissive default config should build"); let spellings = [ - ("power_of_two", LoadBalancerAlgorithm::PowerOfTwo), + ("power_of_n", LoadBalancerAlgorithm::PowerOfN), + ("power_of_two", LoadBalancerAlgorithm::PowerOfN), ("round_robin", LoadBalancerAlgorithm::RoundRobin), ("groq-multiregion", LoadBalancerAlgorithm::WaitAndWiden), ("groq_multiregion", LoadBalancerAlgorithm::WaitAndWiden), @@ -1560,23 +1576,20 @@ fn permissive_default_resolves_alias_and_underscore_spellings() { } #[test] -fn permissive_default_keeps_power_of_two_without_override() { +fn permissive_default_keeps_power_of_n_without_override() { let router = LoadBalancerRouter::from_config(&LoadBalancerConfig::permissive_default()) .expect("permissive default config should build"); let config = router .resolve_algorithm_override("any-model", None) .expect("default algorithm should resolve"); - assert_eq!( - config.config().algorithm(), - LoadBalancerAlgorithm::PowerOfTwo - ); + assert_eq!(config.config().algorithm(), LoadBalancerAlgorithm::PowerOfN); } #[test] fn explicit_config_stays_restrictive() { let router = router_from_json( - r#"{"default":"power-of-two","request_algorithms":{"round-robin":"round-robin"}}"#, + r#"{"default":"power-of-n","request_algorithms":{"round-robin":"round-robin"}}"#, ); let algorithm_override = LoadBalancerAlgorithmOverride::parse("pulsar") .expect("routing algorithm override should parse"); diff --git a/src/libraries/rust/stargate/crates/stargate/src/main.rs b/src/libraries/rust/stargate/crates/stargate/src/main.rs index 9c6efee32..28d4a8594 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/main.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/main.rs @@ -192,7 +192,7 @@ struct Args { /// Skip QUIC TLS certificate verification for outbound connections and relays. #[arg(long, default_value_t = false, env = "STARGATE_QUIC_INSECURE")] quic_insecure: bool, - /// Path to load balancer config JSON file. If omitted, uses power-of-two + /// Path to load balancer config JSON file. If omitted, uses power-of-n /// with every built-in algorithm selectable per request. #[arg(long, value_name = "PATH")] lb_config_path: Option, diff --git a/src/libraries/rust/stargate/crates/stargate/src/routing_state/tests.rs b/src/libraries/rust/stargate/crates/stargate/src/routing_state/tests.rs index a758bc89d..522f417a1 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/routing_state/tests.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/routing_state/tests.rs @@ -2255,7 +2255,7 @@ async fn registered_backend_rtt_means_drive_cluster_load_balancer_selection() { ..LoadBalancerAlgorithmConfig::default() }; let router = LoadBalancerRouter::from_config(&LoadBalancerConfig { - default: LoadBalancerAlgorithm::PowerOfTwo, + default: LoadBalancerAlgorithm::PowerOfN, request_algorithms: HashMap::new(), models: HashMap::from([( model_id.to_string(), diff --git a/src/libraries/rust/stargate/crates/stargate/tests/suite/integration.rs b/src/libraries/rust/stargate/crates/stargate/tests/suite/integration.rs index 5d15d9efb..f0469446a 100644 --- a/src/libraries/rust/stargate/crates/stargate/tests/suite/integration.rs +++ b/src/libraries/rust/stargate/crates/stargate/tests/suite/integration.rs @@ -269,7 +269,7 @@ async fn round_robin_load_balancing() { let mut tmp_file = tempfile::NamedTempFile::new().expect("failed to create temp file"); write!( tmp_file, - r#"{{"default": "power-of-two", "models": {{"rr-model": "round-robin"}}}}"# + r#"{{"default": "power-of-n", "models": {{"rr-model": "round-robin"}}}}"# ) .expect("failed to write config"); let config_path = tmp_file.path().to_str().unwrap().to_string(); diff --git a/src/libraries/rust/stargate/crates/stargate/tests/suite/load_balancing.rs b/src/libraries/rust/stargate/crates/stargate/tests/suite/load_balancing.rs index c78b1edde..f52a4fb56 100644 --- a/src/libraries/rust/stargate/crates/stargate/tests/suite/load_balancing.rs +++ b/src/libraries/rust/stargate/crates/stargate/tests/suite/load_balancing.rs @@ -403,7 +403,7 @@ fn float_eq(actual: f64, expected: f64) -> bool { } #[tokio::test] -async fn power_of_two_prefers_less_input_work() { +async fn power_of_n_prefers_less_input_work() { let stargate = RunningStargate::start("test-sg-p2c", None).await; let mut low = RegisteredBackend::active(stargate.grpc_addr, "p2c-model", "inst-low-headroom").await; @@ -476,7 +476,7 @@ async fn input_work_admission_rejects_overloaded_pool_and_registered_unavailable let stargate = RunningStargate::start( "test-sg-input-work-admission", Some( - r#"{"models": {"admission-model": {"algorithm": "power-of-two", "max_input_work_seconds": 0.5}}}"#, + r#"{"models": {"admission-model": {"algorithm": "power-of-n", "max_input_work_seconds": 0.5}}}"#, ), ) .await; @@ -655,7 +655,7 @@ async fn random_load_balancing_uses_all_instances() { } #[tokio::test] -async fn power_of_two_uses_cluster_aggregated_metrics_and_backend_round_robin() { +async fn power_of_n_uses_cluster_aggregated_metrics_and_backend_round_robin() { let stargate = RunningStargate::start("test-sg-p2c-clusters", None).await; let state = stargate.handle.state(); let mut shared_a = RegisteredBackend::active_in_cluster( @@ -850,9 +850,7 @@ async fn power_of_two_uses_cluster_aggregated_metrics_and_backend_round_robin() async fn wait_and_widen_load_balancing_prefers_lower_estimated_ttft() { let stargate = RunningStargate::start( "test-sg-wait-and-widen", - Some( - r#"{"default": "power-of-two", "models": {"wait-and-widen-model": "wait-and-widen"}}"#, - ), + Some(r#"{"default": "power-of-n", "models": {"wait-and-widen-model": "wait-and-widen"}}"#), ) .await; @@ -923,7 +921,7 @@ async fn wait_and_widen_waits_for_later_bucket_when_fastest_is_full() { let stargate = RunningStargate::start( "test-sg-wait-and-widen-wait", Some( - r#"{"default": "power-of-two", "models": {"wait-and-widen-wait-model": "wait-and-widen"}}"#, + r#"{"default": "power-of-n", "models": {"wait-and-widen-wait-model": "wait-and-widen"}}"#, ), ) .await; @@ -991,7 +989,7 @@ async fn wait_and_widen_cache_affinity_prefers_stable_subset_then_falls_back() { "test-sg-wait-and-widen-affinity", Some( r#"{ - "default": "power-of-two", + "default": "power-of-n", "models": { "wait-and-widen-affinity-model": { "algorithm": "wait-and-widen", @@ -1123,7 +1121,7 @@ async fn wait_and_widen_requires_cache_affinity_key_when_configured() { "test-sg-wait-and-widen-affinity-required", Some( r#"{ - "default": "power-of-two", + "default": "power-of-n", "models": { "wait-and-widen-affinity-required-model": { "algorithm": "wait-and-widen", @@ -1204,7 +1202,7 @@ async fn wait_and_widen_priority_header_uses_matching_queue_estimate() { "test-sg-wait-and-widen-priority", Some( r#"{ - "default": "power-of-two", + "default": "power-of-n", "models": { "wait-and-widen-priority-model": { "algorithm": "wait-and-widen", @@ -1311,7 +1309,7 @@ async fn pulsar_routes_same_affinity_key_consistently() { "test-sg-pulsar", Some( r#"{ - "default": "power-of-two", + "default": "power-of-n", "models": { "pulsar-model": { "algorithm": "pulsar", diff --git a/src/libraries/rust/stargate/crates/stargate/tests/suite/proxy_contract.rs b/src/libraries/rust/stargate/crates/stargate/tests/suite/proxy_contract.rs index 54096d918..407851c39 100644 --- a/src/libraries/rust/stargate/crates/stargate/tests/suite/proxy_contract.rs +++ b/src/libraries/rust/stargate/crates/stargate/tests/suite/proxy_contract.rs @@ -1009,7 +1009,7 @@ impl PulsarHeaderFixture { let mut config_file = tempfile::NamedTempFile::new().expect("failed to create temp file"); std::io::Write::write_all( &mut config_file, - br#"{"default":"power-of-two","models":{"pulsar-model":{"algorithm":"pulsar","seed":"test-seed","require_cache_affinity_key":true,"require_input_tokens":true}}}"#, + br#"{"default":"power-of-n","models":{"pulsar-model":{"algorithm":"pulsar","seed":"test-seed","require_cache_affinity_key":true,"require_input_tokens":true}}}"#, ) .expect("failed to write config"); let config_path = config_file.path().to_str().unwrap().to_string(); @@ -2027,7 +2027,7 @@ async fn transport_local_shared_cluster_failover_stays_within_selected_cluster() init_crypto(); let mut tmp_file = tempfile::NamedTempFile::new().expect("failed to create temp file"); - std::io::Write::write_all(&mut tmp_file, br#"{"default":"power-of-two"}"#) + std::io::Write::write_all(&mut tmp_file, br#"{"default":"power-of-n"}"#) .expect("failed to write config"); let config_path = tmp_file.path().to_str().unwrap().to_string(); @@ -2597,7 +2597,7 @@ async fn queue_estimate_mismatch_retries_sibling_in_selected_shared_cluster() { ); assert_delta!( &before_metrics, &metrics, "stargate_routing_selections_total", 1.0; - r#"algorithm="power-of-two""#, + r#"algorithm="power-of-n""#, r#"model="queue-mismatch-shared-model""#, r#"routing_key="""#, r#"selection="primary""# diff --git a/src/libraries/rust/stargate/docs/diagrams/chat-completions-e2e.puml b/src/libraries/rust/stargate/docs/diagrams/chat-completions-e2e.puml index 9682b0430..f1aec1ff0 100644 --- a/src/libraries/rust/stargate/docs/diagrams/chat-completions-e2e.puml +++ b/src/libraries/rust/stargate/docs/diagrams/chat-completions-e2e.puml @@ -35,7 +35,7 @@ Proxy -> LB: cluster_candidates_for_target(\n RoutingTargetKey{my-rk, my-model} LB --> Proxy: [cluster-A (rtt=5ms), cluster-B (rtt=12ms)] note over LB - Default: power-of-two + Default: power-of-n (sample 2, pick lower prompt-work time). Only clusters with a backend that has an open QUIC connection and a healthy RTT diff --git a/src/libraries/rust/stargate/docs/load-balancer-configuration.md b/src/libraries/rust/stargate/docs/load-balancer-configuration.md index de8329e8f..5aeca7fe1 100644 --- a/src/libraries/rust/stargate/docs/load-balancer-configuration.md +++ b/src/libraries/rust/stargate/docs/load-balancer-configuration.md @@ -4,7 +4,7 @@ Stargate selects one load-balancing algorithm for each model. A request can select another preconfigured algorithm through a trusted header. This page defines the `lb-config.json` schema and the behavior of -`power-of-two`, `wait-and-widen`, `pulsar`, and `pulsar-wait-and-widen`. +`power-of-n`, `wait-and-widen`, `pulsar`, and `pulsar-wait-and-widen`. Deployment systems own the file mount and the `--lb-config-path` argument. ## Load the configuration @@ -15,7 +15,7 @@ Start Stargate with an optional JSON file: --lb-config-path=/config/lb-config.json ``` -When the argument is absent, Stargate uses `power-of-two` for every model and +When the argument is absent, Stargate uses `power-of-n` for every model and accepts a routing-method override when it is in the allowlist of built-in algorithms, each with its default settings. When the argument is present, the file defines the allowlist. @@ -30,24 +30,25 @@ The top-level object has three fields: | Field | Type | Default | Meaning | | --- | --- | --- | --- | -| `default` | algorithm name | `power-of-two` | Algorithm for models without an entry in `models`. | +| `default` | algorithm name | `power-of-n` | Algorithm for models without an entry in `models`. | | `request_algorithms` | object | `{}` | Algorithms that `x-routing-method` may select for every model. | | `models` | object | `{}` | Exact model ID to algorithm configuration. | -Valid algorithm names are `power-of-two`, `wait-and-widen`, `round-robin`, +Valid algorithm names are `power-of-n`, `wait-and-widen`, `round-robin`, `random`, `pulsar`, and `pulsar-wait-and-widen`. -For backward compatibility, Stargate also accepts `groq-multiregion` as an -alias for `wait-and-widen` and `pulsar-multiregion` as an alias for -`pulsar-wait-and-widen` in `default`, `models`, `request_algorithms`, detailed -algorithm objects, and routing-method overrides. Use the canonical names for -new configurations. +For backward compatibility, Stargate also accepts `powerOfN`, `powerOf2`, and +`power-of-two` as aliases for `power-of-n`, `groq-multiregion` as an alias for +`wait-and-widen`, and `pulsar-multiregion` as an alias for +`pulsar-wait-and-widen`. These aliases work in `default`, `models`, +`request_algorithms`, detailed algorithm objects, and routing-method overrides. +Use the canonical names for new configurations. An entry in `models` or `request_algorithms` can be an algorithm name: ```json { - "default": "power-of-two", + "default": "power-of-n", "models": { "model-a": "wait-and-widen" } @@ -58,7 +59,7 @@ Use a detailed object to set algorithm fields: ```json { - "default": "power-of-two", + "default": "power-of-n", "models": { "model-a": { "algorithm": "wait-and-widen", @@ -111,13 +112,13 @@ Choose based on the routing goal and available backend statistics: | Keep the same prefix on a stable, capacity-weighted cluster. | `pulsar` | Positive finite `last_mean_input_tps` for every participating cluster. | | Keep Pulsar affinity when possible, but escape to lower-latency capacity when the primary cannot meet queue policy. | `pulsar-wait-and-widen` | Pulsar capacity plus the RTT and queue statistics used by `wait-and-widen`. | -Use `power-of-two` when these statistics or affinity requirements are not +Use `power-of-n` when these statistics or affinity requirements are not available. Use `round-robin` for deterministic cycling and `random` for uniform random selection. -## `power-of-two` +## `power-of-n` -`power-of-two` uniformly samples distinct eligible clusters and selects the +`power-of-n` uniformly samples distinct eligible clusters and selects the cluster with the lowest sum of queued and request input tokens divided by its last mean input TPS. It breaks equal scores randomly. Retried clusters are excluded before sampling. @@ -129,10 +130,10 @@ compares every eligible cluster once. ```json { - "default": "power-of-two", + "default": "power-of-n", "models": { "model-a": { - "algorithm": "power-of-two", + "algorithm": "power-of-n", "sample_count": 4 } } @@ -167,7 +168,7 @@ Minimal configuration: ```json { - "default": "power-of-two", + "default": "power-of-n", "models": { "model-a": { "algorithm": "wait-and-widen", @@ -194,7 +195,7 @@ Minimal configuration: ```json { - "default": "power-of-two", + "default": "power-of-n", "models": { "model-a": { "algorithm": "pulsar", @@ -218,7 +219,7 @@ Minimal configuration: ```json { - "default": "power-of-two", + "default": "power-of-n", "models": { "model-a": { "algorithm": "pulsar-wait-and-widen", @@ -233,7 +234,7 @@ Minimal configuration: ## Algorithm fields -`power-of-two` supports this field: +`power-of-n` supports this field: | Field | Type | Default | Constraint and effect | | --- | --- | --- | --- | @@ -288,7 +289,7 @@ Preconfigure every algorithm that a request may select: ```json { - "default": "power-of-two", + "default": "power-of-n", "request_algorithms": { "wait-and-widen": "wait-and-widen", "pulsar": { diff --git a/src/libraries/rust/stargate/docs/multi-backend-clusters.md b/src/libraries/rust/stargate/docs/multi-backend-clusters.md index 4e6051ccf..c75df11db 100644 --- a/src/libraries/rust/stargate/docs/multi-backend-clusters.md +++ b/src/libraries/rust/stargate/docs/multi-backend-clusters.md @@ -104,7 +104,7 @@ source backend disappears, recompute from remaining active snapshots. All algorithms choose from cluster snapshots. -- `power-of-two`: compares aggregated cluster load. +- `power-of-n`: compares aggregated cluster load. - `wait-and-widen`: uses aggregated stats and representative RTT. - `pulsar-wait-and-widen`: combines Pulsar ranking with WaitAndWiden fallback. - `round-robin`: rounds across clusters. From 03627e8d32bae8552610de0a275e54aed4a7ee9c Mon Sep 17 00:00:00 2001 From: Barry Greengus Date: Thu, 13 Aug 2026 19:15:54 +0000 Subject: [PATCH 4/5] fix(stargate): accept power-of-n aliases consistently Relates to #831 --- deploy/helm/llm-request-router/README.md | 2 +- docs/user/cli.md | 4 +-- docs/user/llm-function-enablement.md | 6 +++-- docs/user/llm-gateway.md | 6 ++--- .../user/llm-request-router-load-balancing.md | 17 +++++++------ src/clis/nvcf-cli/README.md | 2 +- src/clis/nvcf-cli/USAGE-GUIDE.md | 3 +-- src/clis/nvcf-cli/cmd/function.go | 12 ++++----- .../nvcf-cli/cmd/function_llm_model_test.go | 16 ++++++------ .../management/dto/LlmConfigValidator.java | 5 +++- .../dto/LlmConfigValidatorTest.java | 4 +-- .../stargate/src/load_balancer/config.rs | 11 +++++++- .../stargate/src/load_balancer/tests.rs | 25 ------------------- 13 files changed, 50 insertions(+), 63 deletions(-) diff --git a/deploy/helm/llm-request-router/README.md b/deploy/helm/llm-request-router/README.md index 3f8d6b15b..b509319b0 100644 --- a/deploy/helm/llm-request-router/README.md +++ b/deploy/helm/llm-request-router/README.md @@ -87,7 +87,7 @@ The chart can pass a Stargate load-balancer config in either of two ways: - `llmRequestRouter.loadBalancer.config` embeds JSON directly in the release. The chart writes it to a ConfigMap and starts Stargate with `--lb-config-path=/etc/llm-request-router/lb-config.json`. - `llmRequestRouter.loadBalancer.configPath` points Stargate at an existing file path and starts it with `--lb-config-path=`. -`config` takes precedence over `configPath` when both are set. If neither value is set, Stargate uses its built-in default algorithm, `power-of-two`. +`config` takes precedence over `configPath` when both are set. If neither value is set, Stargate uses its built-in default algorithm, `power-of-n`, with a sample count of `2`. Existing `power-of-two` and `powerOf2` values remain accepted as aliases. See the [Stargate load balancer configuration](../../../src/libraries/rust/stargate/docs/load-balancer-configuration.md) diff --git a/docs/user/cli.md b/docs/user/cli.md index 16d87a440..effac5b0f 100644 --- a/docs/user/cli.md +++ b/docs/user/cli.md @@ -671,7 +671,7 @@ All `function create` flags: | `--secrets` | Secrets in `name=value` format (repeatable) | | `--tags` | Comma-separated tags | | `--models` | Model artifacts in `name:version:uri` format (repeatable) | -| `--llm-model` | LLM model config in `name=MODEL,uris=URI\|URI,routingMethod=round_robin\|power_of_two\|groq_multiregion\|pulsar\|random,tokenRateLimit=LIMIT` format (repeatable). Token limits use `-` with `S`, `M`, `H`, `D`, or `W`, for example `1000-S`. Use JSON input for combined token limits because inline model specs use commas as field separators. | +| `--llm-model` | LLM model config in `name=MODEL,uris=URI\|URI,routingMethod=round_robin\|power_of_n\|groq_multiregion\|pulsar\|random,tokenRateLimit=LIMIT` format (repeatable). Token limits use `-` with `S`, `M`, `H`, `D`, or `W`, for example `1000-S`. Use JSON input for combined token limits because inline model specs use commas as field separators. | | `--resources` | Resource artifacts in `name:version:uri` format (repeatable) | | `--helm-chart` | Helm chart specification | | `--helm-chart-service` | Helm chart service name | @@ -719,7 +719,7 @@ LLM functions use `functionType: "LLM"` and define model routing metadata under } ``` -For LLM models, `llmConfig.routingMethod` accepts `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random`. +For LLM models, `llmConfig.routingMethod` accepts `round_robin`, `power_of_n`, `groq_multiregion`, `pulsar`, or `random`. Existing `power_of_two`, `power-of-two`, and `powerOf2` values remain accepted as aliases for `power_of_n`. Supported LLM paths are `/v1/chat/completions`, `/v1/responses`, and `/v1/embeddings`. `llmConfig.tokenRateLimit` accepts one or more comma-separated positive integer token limits in `-` format. Supported units are `S` (seconds), `M` (minutes), `H` (hours), `D` (days), and `W` (weeks). Use `1000-S` for a single limit, or `1000-S,5000-M,100000-H,500000-D,1000000-W` for a combined limit with distinct units. Use JSON input for combined limits because inline CLI model specs use commas as field separators. diff --git a/docs/user/llm-function-enablement.md b/docs/user/llm-function-enablement.md index 1903ed500..18d741d98 100644 --- a/docs/user/llm-function-enablement.md +++ b/docs/user/llm-function-enablement.md @@ -78,8 +78,10 @@ API then includes the address in LLM worker configuration. Do not configure the worker address under `api.env`. When the LLM addon is disabled, the stack does not pass a staged endpoint to the API chart. -The request router uses `power-of-two` when no load-balancer configuration is -set, and accepts any supported `routingMethod` from a function. When a +The request router uses `power-of-n` with a sample count of `2` when no +load-balancer configuration is set. Existing `power-of-two` and `powerOf2` +values remain accepted as aliases. The router accepts any supported +`routingMethod` from a function. When a load-balancer configuration is set, a function can only select an algorithm that the configuration enables. diff --git a/docs/user/llm-gateway.md b/docs/user/llm-gateway.md index 863aed2b7..48903359d 100644 --- a/docs/user/llm-gateway.md +++ b/docs/user/llm-gateway.md @@ -63,7 +63,7 @@ Set `functionType` to `LLM` and define model routing metadata under `models[].ll "name": "dummy-model", "llmConfig": { "uris": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"], - "routingMethod": "power_of_two", + "routingMethod": "power_of_n", "tokenRateLimit": "1000-S" } } @@ -79,7 +79,7 @@ Set `functionType` to `LLM` and define model routing metadata under `models[].ll | `/v1/responses` | Supports native Responses API requests. Streaming clients receive server-sent events (SSE). Non-streaming clients receive the terminal Responses JSON object. | | `/v1/embeddings` | Supports embeddings requests with string or string array input. | -`nvcf-cli` accepts `round_robin`, `power_of_two`, `groq_multiregion`, +`nvcf-cli` accepts `round_robin`, `power_of_n`, `groq_multiregion`, `pulsar`, or `random` for `llmConfig.routingMethod`. For the mapping to Stargate algorithms and the request-router allowlist, see @@ -96,7 +96,7 @@ The same configuration can be provided with CLI flags: --inference-url "/" \ --inference-port 8000 \ --function-type LLM \ - --llm-model "name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=power_of_two,tokenRateLimit=1000-S" + --llm-model "name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=power_of_n,tokenRateLimit=1000-S" ``` These per-model routing fields are mutable. Use `nvcf-cli function update --llm-model-update "name=,routingMethod=,tokenRateLimit="` or JSON `modelUpdates` to change them without recreating the function version. diff --git a/docs/user/llm-request-router-load-balancing.md b/docs/user/llm-request-router-load-balancing.md index 2b568affc..d95c72f7d 100644 --- a/docs/user/llm-request-router-load-balancing.md +++ b/docs/user/llm-request-router-load-balancing.md @@ -20,7 +20,7 @@ addons: loadBalancer: config: | { - "default": "power-of-two", + "default": "power-of-n", "request_algorithms": { "round-robin": "round-robin" } @@ -39,7 +39,7 @@ The chart supports two configuration sources: | `loadBalancer.configPath` | The chart only passes `--lb-config-path=`. The operator must add and maintain the file mount by another mechanism. | Inline `config` takes precedence when both values are set. When neither value -is set, Stargate uses its built-in `power-of-two` default and accepts a +is set, Stargate uses its built-in `power-of-n` default and accepts a routing-method override when it is in the allowlist of built-in algorithms. Stargate reads and validates the file only during process startup. The @@ -53,20 +53,21 @@ Algorithm availability is enforced at separate layers: | Layer | Input contract | | --- | --- | -| `lb-config.json` | Canonical Stargate algorithm names: `power-of-two`, `wait-and-widen`, `round-robin`, `random`, `pulsar`, and `pulsar-wait-and-widen`. Legacy `groq-multiregion` and `pulsar-multiregion` aliases remain accepted for existing deployments. | +| `lb-config.json` | Canonical Stargate algorithm names: `power-of-n`, `wait-and-widen`, `round-robin`, `random`, `pulsar`, and `pulsar-wait-and-widen`. The `power-of-two`, `powerOf2`, `powerOfN`, `groq-multiregion`, and `pulsar-multiregion` aliases remain accepted for existing deployments. | | Function model `llmConfig.routingMethod` | The same algorithm names, with underscores accepted in place of hyphens. Legacy aliases remain accepted for existing functions. | | LLM API Gateway | Nonblank routing method from authenticated model metadata, trimmed and forwarded as `x-routing-method` without algorithm validation. | | Stargate `x-routing-method` | Case-insensitive algorithm name with hyphens or underscores. It must match the effective algorithm or a model or top-level `request_algorithms` entry. Otherwise, Stargate returns HTTP `400`. | -For example, when a configuration is set and `power-of-two` is the effective +For example, when a configuration is set and `power-of-n` is the effective algorithm, `wait_and_widen` requires a `wait-and-widen` entry in `request_algorithms`. The legacy `groq_multiregion` value remains accepted and resolves to the same algorithm. -Use `wait-and-widen` and `pulsar-wait-and-widen` in new function metadata, -`lb-config.json` files, request-algorithm maps, and deployment manifests. -Existing `groq-multiregion` and `pulsar-multiregion` values continue to work -through the Stargate and control-plane compatibility aliases. +Use `power-of-n`, `wait-and-widen`, and `pulsar-wait-and-widen` in new function +metadata, `lb-config.json` files, request-algorithm maps, and deployment +manifests. Existing `power-of-two`, `powerOf2`, `groq-multiregion`, and +`pulsar-multiregion` values continue to work through the Stargate and +control-plane compatibility aliases. ## Keep router headers trusted diff --git a/src/clis/nvcf-cli/README.md b/src/clis/nvcf-cli/README.md index 7d043b4c0..252e0ce81 100644 --- a/src/clis/nvcf-cli/README.md +++ b/src/clis/nvcf-cli/README.md @@ -667,7 +667,7 @@ export NVCF_TOKEN="nvapi-your-function-creation-token" `--llm-model` accepts `name`, `uris`, `routingMethod`, and `tokenRateLimit` key/value fields. Separate multiple URIs with `|`. Valid routing -methods are `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, and +methods are `round_robin`, `power_of_n`, `groq_multiregion`, `pulsar`, and `random`; the CLI validates and sends these API/auth spellings in the create request. `tokenRateLimit` supports positive integer token limits for `S`, `M`, `H`, `D`, and `W`. diff --git a/src/clis/nvcf-cli/USAGE-GUIDE.md b/src/clis/nvcf-cli/USAGE-GUIDE.md index 73a369559..8e5b29ac1 100644 --- a/src/clis/nvcf-cli/USAGE-GUIDE.md +++ b/src/clis/nvcf-cli/USAGE-GUIDE.md @@ -813,7 +813,7 @@ curl -X POST https://api.nvcf.nvidia.com/v2/nvcf/accounts/nvcf-default/registry- ``` For LLM models, `llmConfig.routingMethod` accepts the API/auth spellings -`round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random`. The +`round_robin`, `power_of_n`, `groq_multiregion`, `pulsar`, or `random`. The CLI validates these values before sending the create request. Supported LLM paths are `/v1/chat/completions`, `/v1/responses`, and `/v1/embeddings`. @@ -1373,4 +1373,3 @@ nvcf-cli task delete # DELETE - permanent ``` `task delete` clears the saved task from state when it matches. - diff --git a/src/clis/nvcf-cli/cmd/function.go b/src/clis/nvcf-cli/cmd/function.go index 8eff0fa33..4db92cdf1 100644 --- a/src/clis/nvcf-cli/cmd/function.go +++ b/src/clis/nvcf-cli/cmd/function.go @@ -568,7 +568,7 @@ func init() { createCmd.Flags().StringVar(&createFlags.helmChartServiceName, "helm-chart-service", "", "Helm chart service name") createCmd.Flags().StringSliceVar(&createFlags.secrets, "secrets", []string{}, "Secrets in name=value format (e.g., API_KEY=secret123,DB_PASSWORD=pass456)") createCmd.Flags().StringSliceVar(&createFlags.models, "models", []string{}, "Model artifacts (format: name:version:uri)") - createCmd.Flags().StringArrayVar(&createFlags.llmModels, "llm-model", []string{}, "LLM model config (format: name=,uris=|,routingMethod=,tokenRateLimit=)") + createCmd.Flags().StringArrayVar(&createFlags.llmModels, "llm-model", []string{}, "LLM model config (format: name=,uris=|,routingMethod=,tokenRateLimit=)") createCmd.Flags().StringSliceVar(&createFlags.resources, "resources", []string{}, "Resource artifacts (format: name:version:uri)") createCmd.Flags().StringVar(&createFlags.rateLimit, "rate-limit", "", "Rate limit pattern (e.g., '100-S', '50-M', '10-H', '5-D')") createCmd.Flags().StringSliceVar(&createFlags.rateLimitExempted, "rate-limit-exempted", []string{}, "NCA IDs exempted from rate limiting") @@ -609,7 +609,7 @@ func init() { updateCmd.Flags().StringVar(&updateFlags.functionID, "function-id", "", "Function ID (required)") updateCmd.Flags().StringVar(&updateFlags.versionID, "version-id", "", "Version ID (required)") updateCmd.Flags().StringSliceVar(&updateFlags.tags, "tags", []string{}, "Function tags (comma-separated)") - updateCmd.Flags().StringArrayVar(&updateFlags.llmModelUpdates, "llm-model-update", []string{}, "LLM model update (format: name=,routingMethod=,tokenRateLimit=)") + updateCmd.Flags().StringArrayVar(&updateFlags.llmModelUpdates, "llm-model-update", []string{}, "LLM model update (format: name=,routingMethod=,tokenRateLimit=)") } // ============================================================================ @@ -820,10 +820,8 @@ func normalizeLLMRoutingMethod(value string) (string, error) { return "round_robin", nil case "round-robin": return "round_robin", nil - case "power_of_two": - return "power_of_two", nil - case "power-of-two": - return "power_of_two", nil + case "power_of_n", "power-of-n", "powerofn", "power_of_two", "power-of-two", "powerof2": + return "power_of_n", nil case "wait_and_widen", "wait-and-widen": return "wait_and_widen", nil case "pulsar_wait_and_widen", "pulsar-wait-and-widen": @@ -837,7 +835,7 @@ func normalizeLLMRoutingMethod(value string) (string, error) { case "random": return "random", nil default: - return "", fmt.Errorf("unsupported routingMethod %q (expected round_robin, power_of_two, wait_and_widen, pulsar_wait_and_widen, groq_multiregion, pulsar, or random)", value) + return "", fmt.Errorf("unsupported routingMethod %q (expected round_robin, power_of_n, wait_and_widen, pulsar_wait_and_widen, groq_multiregion, pulsar, or random)", value) } } diff --git a/src/clis/nvcf-cli/cmd/function_llm_model_test.go b/src/clis/nvcf-cli/cmd/function_llm_model_test.go index 15dfbd633..a6d29d39c 100644 --- a/src/clis/nvcf-cli/cmd/function_llm_model_test.go +++ b/src/clis/nvcf-cli/cmd/function_llm_model_test.go @@ -140,7 +140,7 @@ func TestCreateConfigModelLLMConfigOmittedWhenNil(t *testing.T) { func TestParseLLMModelString(t *testing.T) { t.Parallel() - model, err := parseLLMModelString("name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=power_of_two,tokenRateLimit=1000-M") + model, err := parseLLMModelString("name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=power_of_n,tokenRateLimit=1000-M") if err != nil { t.Fatalf("parse llm model: %v", err) } @@ -152,8 +152,8 @@ func TestParseLLMModelString(t *testing.T) { t.Fatal("llmConfig is nil") } assertStringSlice(t, model.LLMConfig.URIs, []string{"/v1/chat/completions", "/v1/responses", "/v1/embeddings"}) - if got := stringValue(model.LLMConfig.RoutingMethod); got != "power_of_two" { - t.Fatalf("routingMethod = %q, want power_of_two", got) + if got := stringValue(model.LLMConfig.RoutingMethod); got != "power_of_n" { + t.Fatalf("routingMethod = %q, want power_of_n", got) } } @@ -167,7 +167,7 @@ func TestParseLLMModelStringAcceptsAdvancedRoutingMethods(t *testing.T) { {input: "groq_multiregion", expected: "groq_multiregion"}, {input: "groq-multiregion", expected: "groq_multiregion"}, {input: "round-robin", expected: "round_robin"}, - {input: "power-of-two", expected: "power_of_two"}, + {input: "power_of_n", expected: "power_of_n"}, {input: "wait_and_widen", expected: "wait_and_widen"}, {input: "wait-and-widen", expected: "wait_and_widen"}, {input: "pulsar_wait_and_widen", expected: "pulsar_wait_and_widen"}, @@ -372,7 +372,7 @@ func TestUpdateConfigParsesLLMModelUpdatesFromJSON(t *testing.T) { func TestParseLLMModelUpdateString(t *testing.T) { t.Parallel() - update, err := parseLLMModelUpdateString("name=dummy-model,routingMethod=power_of_two,tokenRateLimit=1000-M") + update, err := parseLLMModelUpdateString("name=dummy-model,routingMethod=power_of_n,tokenRateLimit=1000-M") if err != nil { t.Fatalf("parse llm model update: %v", err) } @@ -383,8 +383,8 @@ func TestParseLLMModelUpdateString(t *testing.T) { if update.LLMConfig == nil { t.Fatal("llmConfig is nil") } - if got := stringValue(update.LLMConfig.RoutingMethod); got != "power_of_two" { - t.Fatalf("routingMethod = %q, want power_of_two", got) + if got := stringValue(update.LLMConfig.RoutingMethod); got != "power_of_n" { + t.Fatalf("routingMethod = %q, want power_of_n", got) } if got := stringValue(update.LLMConfig.TokenRateLimit); got != "1000-M" { t.Fatalf("tokenRateLimit = %q, want 1000-M", got) @@ -401,7 +401,7 @@ func TestParseLLMModelUpdateStringAcceptsAdvancedRoutingMethods(t *testing.T) { {input: "groq_multiregion", expected: "groq_multiregion"}, {input: "groq-multiregion", expected: "groq_multiregion"}, {input: "round-robin", expected: "round_robin"}, - {input: "power-of-two", expected: "power_of_two"}, + {input: "power_of_n", expected: "power_of_n"}, {input: "wait_and_widen", expected: "wait_and_widen"}, {input: "wait-and-widen", expected: "wait_and_widen"}, {input: "pulsar_wait_and_widen", expected: "pulsar_wait_and_widen"}, diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java index a77fb9a9a..3c53db45a 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java @@ -23,7 +23,10 @@ private LlmConfigValidator() {} // Stargate LoadBalancerAlgorithm values; keep in sync. Blank = router default. private static final Set VALID_ROUTING_METHODS = Set.of( + "power-of-n", "power-of-two", + "powerof2", + "powerofn", "wait-and-widen", "round-robin", "random", @@ -39,7 +42,7 @@ private LlmConfigValidator() {} private static final String MESG_INVALID_ROUTING_METHOD = "Invalid request: 'llmConfig.routingMethod' for model '%s' is invalid; supported " - + "values are [power-of-two, wait-and-widen, round-robin, random, pulsar, " + + "values are [power-of-n, wait-and-widen, round-robin, random, pulsar, " + "pulsar-wait-and-widen, groq-multiregion, pulsar-multiregion]"; private static final String MESG_INVALID_TOKEN_RATE_LIMIT = "Invalid request: 'llmConfig.tokenRateLimit' for model '%s' is invalid; expected " diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java index 62afbb9a9..cb39b0531 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java @@ -19,10 +19,10 @@ class LlmConfigValidatorTest { @ParameterizedTest @ValueSource(strings = { - "power-of-two", "wait-and-widen", "round-robin", "random", "pulsar", + "power-of-n", "wait-and-widen", "round-robin", "random", "pulsar", "pulsar-wait-and-widen", "groq-multiregion", "pulsar-multiregion", // Router normalizes case and '_' to '-', so these are accepted too. - "Power-Of-Two", "power_of_two", "wait_and_widen", "pulsar_wait_and_widen", + "Power-Of-N", "power_of_n", "wait_and_widen", "pulsar_wait_and_widen", "groq_multiregion", "pulsar_multiregion", " pulsar " }) void validRoutingMethodsAccepted(String routingMethod) { diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs index 6c4c33260..c9603a5a7 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/config.rs @@ -437,7 +437,16 @@ impl RawCommonAlgorithmConfig { #[derive(Debug, Deserialize)] #[serde(tag = "algorithm", rename_all = "kebab-case")] enum RawLoadBalancerAlgorithmConfig { - #[serde(alias = "power-of-two", alias = "powerOf2", alias = "powerOfN")] + // Technical debt: these aliases duplicate LoadBalancerAlgorithm because serde parses this + // tagged detailed-config enum independently. Keep both lists in sync until NVIDIA/nvcf#831 + // centralizes algorithm-name parsing. + #[serde( + alias = "power-of-two", + alias = "powerOf2", + alias = "powerOfN", + alias = "powerof2", + alias = "powerofn" + )] PowerOfN { #[serde(flatten)] settings: PowerOfNAlgorithmConfig, diff --git a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs index 5009697fb..ee2916ddf 100644 --- a/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs +++ b/src/libraries/rust/stargate/crates/stargate/src/load_balancer/tests.rs @@ -1002,31 +1002,6 @@ fn legacy_algorithm_names_remain_compatible_in_short_configs() { } } -#[test] -fn power_of_n_aliases_remain_compatible() { - for alias in ["power-of-two", "powerOf2", "powerOfN"] { - let config: LoadBalancerConfig = parse_json(&format!(r#"{{"default":"{alias}"}}"#)); - assert_eq!(config.default, LoadBalancerAlgorithm::PowerOfN, "{alias}"); - - let detailed: LoadBalancerAlgorithmConfig = - parse_json(&format!(r#"{{"algorithm":"{alias}"}}"#)); - assert_eq!( - detailed.algorithm(), - LoadBalancerAlgorithm::PowerOfN, - "{alias}" - ); - - let algorithm_override = LoadBalancerAlgorithmOverride::parse(alias) - .expect("power-of-n alias should remain compatible"); - assert_eq!( - algorithm_override.algorithm(), - LoadBalancerAlgorithm::PowerOfN, - "{alias}" - ); - assert_eq!(algorithm_override.requested_algorithm(), alias); - } -} - #[test] fn legacy_algorithm_names_remain_compatible_in_detailed_configs() { let router = router_from_json( From fa0ab3379f57861ccf6f525ae9a0feb941120451 Mon Sep 17 00:00:00 2001 From: Barry Greengus Date: Thu, 13 Aug 2026 19:22:10 +0000 Subject: [PATCH 5/5] refactor(stargate): keep alias fix local Relates to #831 --- deploy/helm/llm-request-router/README.md | 2 +- docs/user/cli.md | 4 ++-- docs/user/llm-function-enablement.md | 6 ++---- docs/user/llm-gateway.md | 6 +++--- docs/user/llm-request-router-load-balancing.md | 17 ++++++++--------- src/clis/nvcf-cli/README.md | 2 +- src/clis/nvcf-cli/USAGE-GUIDE.md | 3 ++- src/clis/nvcf-cli/cmd/function.go | 12 +++++++----- .../nvcf-cli/cmd/function_llm_model_test.go | 16 ++++++++-------- .../management/dto/LlmConfigValidator.java | 5 +---- .../management/dto/LlmConfigValidatorTest.java | 4 ++-- 11 files changed, 37 insertions(+), 40 deletions(-) diff --git a/deploy/helm/llm-request-router/README.md b/deploy/helm/llm-request-router/README.md index b509319b0..3f8d6b15b 100644 --- a/deploy/helm/llm-request-router/README.md +++ b/deploy/helm/llm-request-router/README.md @@ -87,7 +87,7 @@ The chart can pass a Stargate load-balancer config in either of two ways: - `llmRequestRouter.loadBalancer.config` embeds JSON directly in the release. The chart writes it to a ConfigMap and starts Stargate with `--lb-config-path=/etc/llm-request-router/lb-config.json`. - `llmRequestRouter.loadBalancer.configPath` points Stargate at an existing file path and starts it with `--lb-config-path=`. -`config` takes precedence over `configPath` when both are set. If neither value is set, Stargate uses its built-in default algorithm, `power-of-n`, with a sample count of `2`. Existing `power-of-two` and `powerOf2` values remain accepted as aliases. +`config` takes precedence over `configPath` when both are set. If neither value is set, Stargate uses its built-in default algorithm, `power-of-two`. See the [Stargate load balancer configuration](../../../src/libraries/rust/stargate/docs/load-balancer-configuration.md) diff --git a/docs/user/cli.md b/docs/user/cli.md index effac5b0f..16d87a440 100644 --- a/docs/user/cli.md +++ b/docs/user/cli.md @@ -671,7 +671,7 @@ All `function create` flags: | `--secrets` | Secrets in `name=value` format (repeatable) | | `--tags` | Comma-separated tags | | `--models` | Model artifacts in `name:version:uri` format (repeatable) | -| `--llm-model` | LLM model config in `name=MODEL,uris=URI\|URI,routingMethod=round_robin\|power_of_n\|groq_multiregion\|pulsar\|random,tokenRateLimit=LIMIT` format (repeatable). Token limits use `-` with `S`, `M`, `H`, `D`, or `W`, for example `1000-S`. Use JSON input for combined token limits because inline model specs use commas as field separators. | +| `--llm-model` | LLM model config in `name=MODEL,uris=URI\|URI,routingMethod=round_robin\|power_of_two\|groq_multiregion\|pulsar\|random,tokenRateLimit=LIMIT` format (repeatable). Token limits use `-` with `S`, `M`, `H`, `D`, or `W`, for example `1000-S`. Use JSON input for combined token limits because inline model specs use commas as field separators. | | `--resources` | Resource artifacts in `name:version:uri` format (repeatable) | | `--helm-chart` | Helm chart specification | | `--helm-chart-service` | Helm chart service name | @@ -719,7 +719,7 @@ LLM functions use `functionType: "LLM"` and define model routing metadata under } ``` -For LLM models, `llmConfig.routingMethod` accepts `round_robin`, `power_of_n`, `groq_multiregion`, `pulsar`, or `random`. Existing `power_of_two`, `power-of-two`, and `powerOf2` values remain accepted as aliases for `power_of_n`. +For LLM models, `llmConfig.routingMethod` accepts `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random`. Supported LLM paths are `/v1/chat/completions`, `/v1/responses`, and `/v1/embeddings`. `llmConfig.tokenRateLimit` accepts one or more comma-separated positive integer token limits in `-` format. Supported units are `S` (seconds), `M` (minutes), `H` (hours), `D` (days), and `W` (weeks). Use `1000-S` for a single limit, or `1000-S,5000-M,100000-H,500000-D,1000000-W` for a combined limit with distinct units. Use JSON input for combined limits because inline CLI model specs use commas as field separators. diff --git a/docs/user/llm-function-enablement.md b/docs/user/llm-function-enablement.md index 18d741d98..1903ed500 100644 --- a/docs/user/llm-function-enablement.md +++ b/docs/user/llm-function-enablement.md @@ -78,10 +78,8 @@ API then includes the address in LLM worker configuration. Do not configure the worker address under `api.env`. When the LLM addon is disabled, the stack does not pass a staged endpoint to the API chart. -The request router uses `power-of-n` with a sample count of `2` when no -load-balancer configuration is set. Existing `power-of-two` and `powerOf2` -values remain accepted as aliases. The router accepts any supported -`routingMethod` from a function. When a +The request router uses `power-of-two` when no load-balancer configuration is +set, and accepts any supported `routingMethod` from a function. When a load-balancer configuration is set, a function can only select an algorithm that the configuration enables. diff --git a/docs/user/llm-gateway.md b/docs/user/llm-gateway.md index 48903359d..863aed2b7 100644 --- a/docs/user/llm-gateway.md +++ b/docs/user/llm-gateway.md @@ -63,7 +63,7 @@ Set `functionType` to `LLM` and define model routing metadata under `models[].ll "name": "dummy-model", "llmConfig": { "uris": ["/v1/chat/completions", "/v1/responses", "/v1/embeddings"], - "routingMethod": "power_of_n", + "routingMethod": "power_of_two", "tokenRateLimit": "1000-S" } } @@ -79,7 +79,7 @@ Set `functionType` to `LLM` and define model routing metadata under `models[].ll | `/v1/responses` | Supports native Responses API requests. Streaming clients receive server-sent events (SSE). Non-streaming clients receive the terminal Responses JSON object. | | `/v1/embeddings` | Supports embeddings requests with string or string array input. | -`nvcf-cli` accepts `round_robin`, `power_of_n`, `groq_multiregion`, +`nvcf-cli` accepts `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random` for `llmConfig.routingMethod`. For the mapping to Stargate algorithms and the request-router allowlist, see @@ -96,7 +96,7 @@ The same configuration can be provided with CLI flags: --inference-url "/" \ --inference-port 8000 \ --function-type LLM \ - --llm-model "name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=power_of_n,tokenRateLimit=1000-S" + --llm-model "name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=power_of_two,tokenRateLimit=1000-S" ``` These per-model routing fields are mutable. Use `nvcf-cli function update --llm-model-update "name=,routingMethod=,tokenRateLimit="` or JSON `modelUpdates` to change them without recreating the function version. diff --git a/docs/user/llm-request-router-load-balancing.md b/docs/user/llm-request-router-load-balancing.md index d95c72f7d..2b568affc 100644 --- a/docs/user/llm-request-router-load-balancing.md +++ b/docs/user/llm-request-router-load-balancing.md @@ -20,7 +20,7 @@ addons: loadBalancer: config: | { - "default": "power-of-n", + "default": "power-of-two", "request_algorithms": { "round-robin": "round-robin" } @@ -39,7 +39,7 @@ The chart supports two configuration sources: | `loadBalancer.configPath` | The chart only passes `--lb-config-path=`. The operator must add and maintain the file mount by another mechanism. | Inline `config` takes precedence when both values are set. When neither value -is set, Stargate uses its built-in `power-of-n` default and accepts a +is set, Stargate uses its built-in `power-of-two` default and accepts a routing-method override when it is in the allowlist of built-in algorithms. Stargate reads and validates the file only during process startup. The @@ -53,21 +53,20 @@ Algorithm availability is enforced at separate layers: | Layer | Input contract | | --- | --- | -| `lb-config.json` | Canonical Stargate algorithm names: `power-of-n`, `wait-and-widen`, `round-robin`, `random`, `pulsar`, and `pulsar-wait-and-widen`. The `power-of-two`, `powerOf2`, `powerOfN`, `groq-multiregion`, and `pulsar-multiregion` aliases remain accepted for existing deployments. | +| `lb-config.json` | Canonical Stargate algorithm names: `power-of-two`, `wait-and-widen`, `round-robin`, `random`, `pulsar`, and `pulsar-wait-and-widen`. Legacy `groq-multiregion` and `pulsar-multiregion` aliases remain accepted for existing deployments. | | Function model `llmConfig.routingMethod` | The same algorithm names, with underscores accepted in place of hyphens. Legacy aliases remain accepted for existing functions. | | LLM API Gateway | Nonblank routing method from authenticated model metadata, trimmed and forwarded as `x-routing-method` without algorithm validation. | | Stargate `x-routing-method` | Case-insensitive algorithm name with hyphens or underscores. It must match the effective algorithm or a model or top-level `request_algorithms` entry. Otherwise, Stargate returns HTTP `400`. | -For example, when a configuration is set and `power-of-n` is the effective +For example, when a configuration is set and `power-of-two` is the effective algorithm, `wait_and_widen` requires a `wait-and-widen` entry in `request_algorithms`. The legacy `groq_multiregion` value remains accepted and resolves to the same algorithm. -Use `power-of-n`, `wait-and-widen`, and `pulsar-wait-and-widen` in new function -metadata, `lb-config.json` files, request-algorithm maps, and deployment -manifests. Existing `power-of-two`, `powerOf2`, `groq-multiregion`, and -`pulsar-multiregion` values continue to work through the Stargate and -control-plane compatibility aliases. +Use `wait-and-widen` and `pulsar-wait-and-widen` in new function metadata, +`lb-config.json` files, request-algorithm maps, and deployment manifests. +Existing `groq-multiregion` and `pulsar-multiregion` values continue to work +through the Stargate and control-plane compatibility aliases. ## Keep router headers trusted diff --git a/src/clis/nvcf-cli/README.md b/src/clis/nvcf-cli/README.md index 252e0ce81..7d043b4c0 100644 --- a/src/clis/nvcf-cli/README.md +++ b/src/clis/nvcf-cli/README.md @@ -667,7 +667,7 @@ export NVCF_TOKEN="nvapi-your-function-creation-token" `--llm-model` accepts `name`, `uris`, `routingMethod`, and `tokenRateLimit` key/value fields. Separate multiple URIs with `|`. Valid routing -methods are `round_robin`, `power_of_n`, `groq_multiregion`, `pulsar`, and +methods are `round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, and `random`; the CLI validates and sends these API/auth spellings in the create request. `tokenRateLimit` supports positive integer token limits for `S`, `M`, `H`, `D`, and `W`. diff --git a/src/clis/nvcf-cli/USAGE-GUIDE.md b/src/clis/nvcf-cli/USAGE-GUIDE.md index 8e5b29ac1..73a369559 100644 --- a/src/clis/nvcf-cli/USAGE-GUIDE.md +++ b/src/clis/nvcf-cli/USAGE-GUIDE.md @@ -813,7 +813,7 @@ curl -X POST https://api.nvcf.nvidia.com/v2/nvcf/accounts/nvcf-default/registry- ``` For LLM models, `llmConfig.routingMethod` accepts the API/auth spellings -`round_robin`, `power_of_n`, `groq_multiregion`, `pulsar`, or `random`. The +`round_robin`, `power_of_two`, `groq_multiregion`, `pulsar`, or `random`. The CLI validates these values before sending the create request. Supported LLM paths are `/v1/chat/completions`, `/v1/responses`, and `/v1/embeddings`. @@ -1373,3 +1373,4 @@ nvcf-cli task delete # DELETE - permanent ``` `task delete` clears the saved task from state when it matches. + diff --git a/src/clis/nvcf-cli/cmd/function.go b/src/clis/nvcf-cli/cmd/function.go index 4db92cdf1..8eff0fa33 100644 --- a/src/clis/nvcf-cli/cmd/function.go +++ b/src/clis/nvcf-cli/cmd/function.go @@ -568,7 +568,7 @@ func init() { createCmd.Flags().StringVar(&createFlags.helmChartServiceName, "helm-chart-service", "", "Helm chart service name") createCmd.Flags().StringSliceVar(&createFlags.secrets, "secrets", []string{}, "Secrets in name=value format (e.g., API_KEY=secret123,DB_PASSWORD=pass456)") createCmd.Flags().StringSliceVar(&createFlags.models, "models", []string{}, "Model artifacts (format: name:version:uri)") - createCmd.Flags().StringArrayVar(&createFlags.llmModels, "llm-model", []string{}, "LLM model config (format: name=,uris=|,routingMethod=,tokenRateLimit=)") + createCmd.Flags().StringArrayVar(&createFlags.llmModels, "llm-model", []string{}, "LLM model config (format: name=,uris=|,routingMethod=,tokenRateLimit=)") createCmd.Flags().StringSliceVar(&createFlags.resources, "resources", []string{}, "Resource artifacts (format: name:version:uri)") createCmd.Flags().StringVar(&createFlags.rateLimit, "rate-limit", "", "Rate limit pattern (e.g., '100-S', '50-M', '10-H', '5-D')") createCmd.Flags().StringSliceVar(&createFlags.rateLimitExempted, "rate-limit-exempted", []string{}, "NCA IDs exempted from rate limiting") @@ -609,7 +609,7 @@ func init() { updateCmd.Flags().StringVar(&updateFlags.functionID, "function-id", "", "Function ID (required)") updateCmd.Flags().StringVar(&updateFlags.versionID, "version-id", "", "Version ID (required)") updateCmd.Flags().StringSliceVar(&updateFlags.tags, "tags", []string{}, "Function tags (comma-separated)") - updateCmd.Flags().StringArrayVar(&updateFlags.llmModelUpdates, "llm-model-update", []string{}, "LLM model update (format: name=,routingMethod=,tokenRateLimit=)") + updateCmd.Flags().StringArrayVar(&updateFlags.llmModelUpdates, "llm-model-update", []string{}, "LLM model update (format: name=,routingMethod=,tokenRateLimit=)") } // ============================================================================ @@ -820,8 +820,10 @@ func normalizeLLMRoutingMethod(value string) (string, error) { return "round_robin", nil case "round-robin": return "round_robin", nil - case "power_of_n", "power-of-n", "powerofn", "power_of_two", "power-of-two", "powerof2": - return "power_of_n", nil + case "power_of_two": + return "power_of_two", nil + case "power-of-two": + return "power_of_two", nil case "wait_and_widen", "wait-and-widen": return "wait_and_widen", nil case "pulsar_wait_and_widen", "pulsar-wait-and-widen": @@ -835,7 +837,7 @@ func normalizeLLMRoutingMethod(value string) (string, error) { case "random": return "random", nil default: - return "", fmt.Errorf("unsupported routingMethod %q (expected round_robin, power_of_n, wait_and_widen, pulsar_wait_and_widen, groq_multiregion, pulsar, or random)", value) + return "", fmt.Errorf("unsupported routingMethod %q (expected round_robin, power_of_two, wait_and_widen, pulsar_wait_and_widen, groq_multiregion, pulsar, or random)", value) } } diff --git a/src/clis/nvcf-cli/cmd/function_llm_model_test.go b/src/clis/nvcf-cli/cmd/function_llm_model_test.go index a6d29d39c..15dfbd633 100644 --- a/src/clis/nvcf-cli/cmd/function_llm_model_test.go +++ b/src/clis/nvcf-cli/cmd/function_llm_model_test.go @@ -140,7 +140,7 @@ func TestCreateConfigModelLLMConfigOmittedWhenNil(t *testing.T) { func TestParseLLMModelString(t *testing.T) { t.Parallel() - model, err := parseLLMModelString("name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=power_of_n,tokenRateLimit=1000-M") + model, err := parseLLMModelString("name=dummy-model,uris=/v1/chat/completions|/v1/responses|/v1/embeddings,routingMethod=power_of_two,tokenRateLimit=1000-M") if err != nil { t.Fatalf("parse llm model: %v", err) } @@ -152,8 +152,8 @@ func TestParseLLMModelString(t *testing.T) { t.Fatal("llmConfig is nil") } assertStringSlice(t, model.LLMConfig.URIs, []string{"/v1/chat/completions", "/v1/responses", "/v1/embeddings"}) - if got := stringValue(model.LLMConfig.RoutingMethod); got != "power_of_n" { - t.Fatalf("routingMethod = %q, want power_of_n", got) + if got := stringValue(model.LLMConfig.RoutingMethod); got != "power_of_two" { + t.Fatalf("routingMethod = %q, want power_of_two", got) } } @@ -167,7 +167,7 @@ func TestParseLLMModelStringAcceptsAdvancedRoutingMethods(t *testing.T) { {input: "groq_multiregion", expected: "groq_multiregion"}, {input: "groq-multiregion", expected: "groq_multiregion"}, {input: "round-robin", expected: "round_robin"}, - {input: "power_of_n", expected: "power_of_n"}, + {input: "power-of-two", expected: "power_of_two"}, {input: "wait_and_widen", expected: "wait_and_widen"}, {input: "wait-and-widen", expected: "wait_and_widen"}, {input: "pulsar_wait_and_widen", expected: "pulsar_wait_and_widen"}, @@ -372,7 +372,7 @@ func TestUpdateConfigParsesLLMModelUpdatesFromJSON(t *testing.T) { func TestParseLLMModelUpdateString(t *testing.T) { t.Parallel() - update, err := parseLLMModelUpdateString("name=dummy-model,routingMethod=power_of_n,tokenRateLimit=1000-M") + update, err := parseLLMModelUpdateString("name=dummy-model,routingMethod=power_of_two,tokenRateLimit=1000-M") if err != nil { t.Fatalf("parse llm model update: %v", err) } @@ -383,8 +383,8 @@ func TestParseLLMModelUpdateString(t *testing.T) { if update.LLMConfig == nil { t.Fatal("llmConfig is nil") } - if got := stringValue(update.LLMConfig.RoutingMethod); got != "power_of_n" { - t.Fatalf("routingMethod = %q, want power_of_n", got) + if got := stringValue(update.LLMConfig.RoutingMethod); got != "power_of_two" { + t.Fatalf("routingMethod = %q, want power_of_two", got) } if got := stringValue(update.LLMConfig.TokenRateLimit); got != "1000-M" { t.Fatalf("tokenRateLimit = %q, want 1000-M", got) @@ -401,7 +401,7 @@ func TestParseLLMModelUpdateStringAcceptsAdvancedRoutingMethods(t *testing.T) { {input: "groq_multiregion", expected: "groq_multiregion"}, {input: "groq-multiregion", expected: "groq_multiregion"}, {input: "round-robin", expected: "round_robin"}, - {input: "power_of_n", expected: "power_of_n"}, + {input: "power-of-two", expected: "power_of_two"}, {input: "wait_and_widen", expected: "wait_and_widen"}, {input: "wait-and-widen", expected: "wait_and_widen"}, {input: "pulsar_wait_and_widen", expected: "pulsar_wait_and_widen"}, diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java index 3c53db45a..a77fb9a9a 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/main/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidator.java @@ -23,10 +23,7 @@ private LlmConfigValidator() {} // Stargate LoadBalancerAlgorithm values; keep in sync. Blank = router default. private static final Set VALID_ROUTING_METHODS = Set.of( - "power-of-n", "power-of-two", - "powerof2", - "powerofn", "wait-and-widen", "round-robin", "random", @@ -42,7 +39,7 @@ private LlmConfigValidator() {} private static final String MESG_INVALID_ROUTING_METHOD = "Invalid request: 'llmConfig.routingMethod' for model '%s' is invalid; supported " - + "values are [power-of-n, wait-and-widen, round-robin, random, pulsar, " + + "values are [power-of-two, wait-and-widen, round-robin, random, pulsar, " + "pulsar-wait-and-widen, groq-multiregion, pulsar-multiregion]"; private static final String MESG_INVALID_TOKEN_RATE_LIMIT = "Invalid request: 'llmConfig.tokenRateLimit' for model '%s' is invalid; expected " diff --git a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java index cb39b0531..62afbb9a9 100644 --- a/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java +++ b/src/control-plane-services/cloud-functions/nvcf-core/src/test/java/com/nvidia/nvcf/rest/function/management/dto/LlmConfigValidatorTest.java @@ -19,10 +19,10 @@ class LlmConfigValidatorTest { @ParameterizedTest @ValueSource(strings = { - "power-of-n", "wait-and-widen", "round-robin", "random", "pulsar", + "power-of-two", "wait-and-widen", "round-robin", "random", "pulsar", "pulsar-wait-and-widen", "groq-multiregion", "pulsar-multiregion", // Router normalizes case and '_' to '-', so these are accepted too. - "Power-Of-N", "power_of_n", "wait_and_widen", "pulsar_wait_and_widen", + "Power-Of-Two", "power_of_two", "wait_and_widen", "pulsar_wait_and_widen", "groq_multiregion", "pulsar_multiregion", " pulsar " }) void validRoutingMethodsAccepted(String routingMethod) {