diff --git a/experimental/sgl-router/README.md b/experimental/sgl-router/README.md index 04e84542bee0..bf9f60a1ec38 100644 --- a/experimental/sgl-router/README.md +++ b/experimental/sgl-router/README.md @@ -88,6 +88,14 @@ buckets and are not accepted on this path. Omitting `--chat-routing` keeps the existing policies and defaults. +### Optional tokenizer for load-only routing + +`--no-tokenizer` skips tokenizer loading for load-only policies such as +`power_of_two` and `session_aware`, on either routing path. Workers tokenize the +original messages, and `/v1/tokenize` and `/v1/detokenize` are unavailable. +Cache-aware routing, prefix-cache terms or filters, and `--bucket-config` still +require a tokenizer. + ### Fleet-wide sampling contract `--override-sampling-params` fixes the sampling configuration for every client diff --git a/experimental/sgl-router/src/buckets_reorg.rs b/experimental/sgl-router/src/buckets_reorg.rs index 9d9a1b151073..ef438435f708 100644 --- a/experimental/sgl-router/src/buckets_reorg.rs +++ b/experimental/sgl-router/src/buckets_reorg.rs @@ -164,6 +164,21 @@ impl Bucket { Ok(()) } + /// Whether choosing or serving this bucket reads request tokens: a length + /// bound, or a group policy that matches prefixes. + pub fn needs_request_tokens(&self) -> bool { + let groups: &[&EngineGroup] = match &self.groups { + BucketGroups::Plain(group) => &[group], + BucketGroups::Pd { prefill, decode } => &[prefill, decode], + }; + self.limits.min.is_some() + || self.limits.max.is_some() + || self.max_context_tokens.is_some() + || groups + .iter() + .any(|group| group.policy.needs_request_tokens()) + } + /// Select this bucket's plain engine or complete P/D pair, without dispatching. /// A failed group reports its stage; the caller may then try another bucket. pub async fn pick_engines( @@ -268,6 +283,11 @@ impl BucketResolver { }) } + /// Whether any bucket reads request tokens; startup rejects this under `--no-tokenizer`. + pub fn needs_request_tokens(&self) -> bool { + self.buckets.iter().any(Bucket::needs_request_tokens) + } + /// Return all length-compatible buckets, ordered by unmet SLO preferences, /// then input capacity, rank, and ID. Both preferences have equal weight. /// The caller tries their groups in order until a complete engine selection succeeds. @@ -317,3 +337,59 @@ impl BucketResolver { Ok(buckets) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::policies_reorg::power_of_two::PowerOfTwoPolicy; + use crate::state::load_monitor::engine_reported_load::EngineReportedLoadTable; + + /// A bucket whose group policy reads no tokens, so only the bucket's own + /// length fields can make it token-hungry. + fn load_only_bucket(id: &str) -> Bucket { + let policy: Arc = + Arc::new(PowerOfTwoPolicy::new(EngineReportedLoadTable::new())); + Bucket::new(id, BucketGroups::Plain(EngineGroup::new(policy))) + } + + /// `reorg::chat_completions` gates request tokenization on this predicate, so + /// every length field [`Bucket::fits`] reads has to keep it true. Drop one and + /// a length-bounded bucket would silently select on the request-size estimate + /// that [`crate::server::routes::chat`] falls back to without tokens. + #[test] + fn a_bucket_needs_tokens_for_every_length_field_it_selects_on() { + type Mutate = fn(&mut Bucket); + assert!( + !load_only_bucket("b").needs_request_tokens(), + "a load-only bucket selects on nothing that needs tokens" + ); + for (field, mutate) in [ + ( + "limits.min", + (|b: &mut Bucket| b.limits.min = Some(1)) as Mutate, + ), + ("limits.max", |b: &mut Bucket| b.limits.max = Some(1)), + ("max_context_tokens", |b: &mut Bucket| { + b.max_context_tokens = Some(1) + }), + ] { + let mut bucket = load_only_bucket("b"); + mutate(&mut bucket); + assert!(bucket.needs_request_tokens(), "{field}"); + } + } + + /// One bounded bucket makes every request to the model tokenize: the bucket is + /// chosen from the token count, so it cannot be known to be irrelevant first. + #[test] + fn a_resolver_needs_tokens_when_any_bucket_does() { + let mut bounded = load_only_bucket("bounded"); + bounded.limits.max = Some(1); + assert!(!BucketResolver::new(vec![load_only_bucket("a")]) + .unwrap() + .needs_request_tokens()); + assert!(BucketResolver::new(vec![load_only_bucket("a"), bounded]) + .unwrap() + .needs_request_tokens()); + } +} diff --git a/experimental/sgl-router/src/config/cli.rs b/experimental/sgl-router/src/config/cli.rs index a7f363a72834..22745e27db05 100644 --- a/experimental/sgl-router/src/config/cli.rs +++ b/experimental/sgl-router/src/config/cli.rs @@ -67,6 +67,11 @@ pub struct ModelArgs { #[arg(long)] pub tokenizer_path: Option, + /// Skip loading a tokenizer; workers tokenize requests. Load-only policies only: + /// cache-aware routing, prefix-cache terms or filters, and length buckets require one. + #[arg(long, conflicts_with = "tokenizer_path")] + pub no_tokenizer: bool, + /// Disable generated input_ids; workers tokenize messages, while routing still renders locally. /// Use for worker parser/template overrides or template stop strings. #[arg(long)] @@ -473,10 +478,11 @@ impl Cli { log_format: self.server.log_format, }, model: ModelConfig { - tokenizer_path: self - .model - .tokenizer_path - .unwrap_or_else(|| self.model.model_id.clone()), + tokenizer_path: (!self.model.no_tokenizer).then(|| { + self.model + .tokenizer_path + .unwrap_or_else(|| self.model.model_id.clone()) + }), id: self.model.model_id, disable_input_ids_forwarding: self.model.disable_input_ids_forwarding, tokenizer: TokenizerConfig { @@ -1195,7 +1201,7 @@ mod tests { ]) .unwrap(); assert_eq!(c.model.id, "Qwen/Qwen3-0.6B"); - assert_eq!(c.model.tokenizer_path, "Qwen/Qwen3-0.6B"); + assert_eq!(c.model.tokenizer_path.as_deref(), Some("Qwen/Qwen3-0.6B")); } #[test] @@ -1209,7 +1215,25 @@ mod tests { "http://x:30000", ]) .unwrap(); - assert_eq!(c.model.tokenizer_path, "/models/qwen3/tokenizer.json"); + assert_eq!( + c.model.tokenizer_path.as_deref(), + Some("/models/qwen3/tokenizer.json") + ); + } + + #[test] + fn no_tokenizer_disables_loading_and_conflicts_with_a_path() { + let base = ["--model-id", "qwen3", "--worker-urls", "http://x:30000"]; + let c = into_config(&[&base[..], &["--no-tokenizer"]].concat()).unwrap(); + assert_eq!(c.model.tokenizer_path, None); + assert!(into_config( + &[ + &base[..], + &["--no-tokenizer", "--tokenizer-path", "/t.json"] + ] + .concat() + ) + .is_err()); } #[test] diff --git a/experimental/sgl-router/src/config/mod.rs b/experimental/sgl-router/src/config/mod.rs index 19a12c1aeff1..31af6096c4a2 100644 --- a/experimental/sgl-router/src/config/mod.rs +++ b/experimental/sgl-router/src/config/mod.rs @@ -44,6 +44,11 @@ impl Config { /// Validate invariants not enforced by the CLI parser. pub(crate) fn validate(&self) -> Result<()> { ensure!(!self.model.id.is_empty(), "model id must be non-empty"); + // Policies that read the prompt are checked against the built routing at startup. + ensure!( + self.model.tokenizer_path.is_some() || self.model.bucket_config.is_none(), + "--no-tokenizer is incompatible with --bucket-config" + ); if let Some(bucket_config) = self.model.bucket_config.as_ref() { validate_bucket_config(bucket_config)?; } @@ -243,7 +248,7 @@ mod tests { observability: ObservabilityConfig::default(), model: ModelConfig { id: model_id.into(), - tokenizer_path: "/tmp/tok.json".into(), + tokenizer_path: Some("/tmp/tok.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, @@ -271,6 +276,20 @@ mod tests { cfg("qwen3", &["http://10.0.0.1:30000"]).validate().unwrap(); } + #[test] + fn disabled_tokenizer_rejects_length_buckets() { + let mut c = cfg("qwen3", &["http://10.0.0.1:30000"]); + c.model.tokenizer_path = None; + c.validate().unwrap(); + c.model.bucket_config = Some(BucketConfig { + buckets: vec![], + ttft_slo_policy: SloBucketPolicy::Disabled, + tps_slo_policy: SloBucketPolicy::Disabled, + }); + let err = c.validate().unwrap_err().to_string(); + assert!(err.contains("--no-tokenizer"), "{err}"); + } + #[test] fn rejects_empty_model_id() { let err = cfg("", &["http://10.0.0.1:30000"]) diff --git a/experimental/sgl-router/src/config/types.rs b/experimental/sgl-router/src/config/types.rs index 9fc39b664b3a..09be6f6c4cac 100644 --- a/experimental/sgl-router/src/config/types.rs +++ b/experimental/sgl-router/src/config/types.rs @@ -327,8 +327,8 @@ impl Default for ObservabilityConfig { pub struct ModelConfig { pub id: String, /// Local tokenizer.json or HuggingFace repo id; defaults to `id`. - /// Resolved by [`crate::tokenizer::adapter::load`]. - pub tokenizer_path: String, + /// Resolved by [`crate::tokenizer::adapter::load`]; `None` (`--no-tokenizer`) disables it. + pub tokenizer_path: Option, /// Disable router-generated input IDs for this model; keep routing tokenization. /// Use when workers have rendering defaults or template stops the router cannot see. pub disable_input_ids_forwarding: bool, diff --git a/experimental/sgl-router/src/main.rs b/experimental/sgl-router/src/main.rs index bccf57bd0dd6..b67cf96386b6 100644 --- a/experimental/sgl-router/src/main.rs +++ b/experimental/sgl-router/src/main.rs @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors // SPDX-License-Identifier: Apache-2.0 -use anyhow::{Context, Result}; +use anyhow::{ensure, Context, Result}; use clap::Parser; use sgl_kv_indexer::{GrpcPrefixIndex, PrefixIndex, PrefixIndexConfig}; use sgl_router::{ @@ -107,6 +107,13 @@ async fn main() -> Result<()> { } }; + // Ask the built policies, so any term or filter that reads the prompt is covered. + ensure!( + config.model.tokenizer_path.is_some() + || !chat_routing.needs_request_tokens(&routing_policies), + "--no-tokenizer is incompatible with cache-aware routing and prefix-cache terms or filters" + ); + // Track this router's local view of in-flight requests. let (local_inflight_requests, inflight_cleanup) = start_local_inflight_tracker(&config); diff --git a/experimental/sgl-router/src/policies/factory.rs b/experimental/sgl-router/src/policies/factory.rs index 65f0f155c550..144e790cdb9a 100644 --- a/experimental/sgl-router/src/policies/factory.rs +++ b/experimental/sgl-router/src/policies/factory.rs @@ -336,7 +336,7 @@ mod tests { observability: Default::default(), model: ModelConfig { id: id.into(), - tokenizer_path: "/tmp/x".into(), + tokenizer_path: Some("/tmp/x".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy, @@ -359,6 +359,44 @@ mod tests { } } + #[test] + fn only_prefix_reading_policies_need_request_tokens() { + fn fuse(model: &mut ModelConfig, kinds: &[ScoreTermKind]) { + model.policy = PolicyKind::FusedScore; + model.fused = Some( + (kinds.iter()) + .map(|&kind| crate::config::FusedTerm { kind, weight: None }) + .collect(), + ); + } + type Mutate = fn(&mut ModelConfig); + let cases: [(Mutate, bool); 8] = [ + (|_| {}, false), + (|m| m.policy = PolicyKind::PowerOfTwo, false), + (|m| m.policy = PolicyKind::SessionAware, false), + (|m| m.policy = PolicyKind::CacheAware, true), + (|m| fuse(m, &crate::config::DEFAULT_FUSE), true), + (|m| fuse(m, &[ScoreTermKind::LoadBased]), false), + (|m| fuse(m, &[ScoreTermKind::PrefixCache]), true), + ( + |m| { + m.eligibility = Some(EligibilityConfig { + filters: vec![FilterKind::PrefixCache], + min_prefix_share: Some(0.6), + ..Default::default() + }) + }, + true, + ), + ]; + for (i, (mutate, needs)) in cases.into_iter().enumerate() { + let mut cfg = cfg_with_model("modelA", PolicyKind::RoundRobin); + mutate(&mut cfg.model); + let registry = build_registry_with_defaults(&cfg).unwrap(); + assert_eq!(registry.needs_request_tokens(), needs, "case {i}"); + } + } + #[test] fn build_policy_kind_only_covers_all_variants() { for (kind, needs_load_snapshot, needs_dispatch_timestamps) in [ diff --git a/experimental/sgl-router/src/policies/mod.rs b/experimental/sgl-router/src/policies/mod.rs index a154cb16c5c7..487a8634e263 100644 --- a/experimental/sgl-router/src/policies/mod.rs +++ b/experimental/sgl-router/src/policies/mod.rs @@ -572,6 +572,13 @@ impl PolicyRegistry { self.by_model.get(model).map(|p| p.clone()) } + /// Whether any registered policy reads request tokens; startup rejects this under `--no-tokenizer`. + pub fn needs_request_tokens(&self) -> bool { + self.by_model + .iter() + .any(|entry| entry.value().needs_request_tokens()) + } + /// Attaches metrics to each registered policy. pub fn attach_metrics(&self, metrics: Arc) { for entry in self.by_model.iter() { diff --git a/experimental/sgl-router/src/policies_reorg/cache_aware.rs b/experimental/sgl-router/src/policies_reorg/cache_aware.rs index a5b7b2a964e5..7f051b5c2eb3 100644 --- a/experimental/sgl-router/src/policies_reorg/cache_aware.rs +++ b/experimental/sgl-router/src/policies_reorg/cache_aware.rs @@ -401,6 +401,10 @@ impl Policy for CacheAwarePolicy { stage != Stage::Decode } + fn needs_request_tokens(&self) -> bool { + true + } + fn pick<'a>( &'a self, engines: &'a [Arc], diff --git a/experimental/sgl-router/src/policies_reorg/mod.rs b/experimental/sgl-router/src/policies_reorg/mod.rs index 042c56fe6fc7..b68d0f617637 100644 --- a/experimental/sgl-router/src/policies_reorg/mod.rs +++ b/experimental/sgl-router/src/policies_reorg/mod.rs @@ -96,6 +96,12 @@ pub trait Policy: Send + Sync + Debug { true } + /// Whether picks read request tokens (prefix matching). Load-only policies + /// work from the request-size estimate when no tokenizer is loaded. + fn needs_request_tokens(&self) -> bool { + false + } + /// Runs on a miss within the same candidates; never on an admission rejection. fn fallback(&self) -> Option<&dyn Policy> { None diff --git a/experimental/sgl-router/src/server/app_context.rs b/experimental/sgl-router/src/server/app_context.rs index e3c01e3064bc..5e14e9de0de2 100644 --- a/experimental/sgl-router/src/server/app_context.rs +++ b/experimental/sgl-router/src/server/app_context.rs @@ -38,6 +38,16 @@ pub enum ChatRouting { Reorg(HashMap), } +impl ChatRouting { + /// Whether the built routing reads request tokens; `legacy` holds the legacy policies. + pub fn needs_request_tokens(&self, legacy: &PolicyRegistry) -> bool { + match self { + Self::Legacy => legacy.needs_request_tokens(), + Self::Reorg(resolvers) => resolvers.values().any(BucketResolver::needs_request_tokens), + } + } +} + pub struct AppContext { pub config: Config, pub tokenizers: Arc, @@ -197,7 +207,7 @@ impl AppContext { observability: Default::default(), model: crate::config::ModelConfig { id: "stub-model".into(), - tokenizer_path: "stub".into(), + tokenizer_path: Some("stub".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: crate::config::PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/src/server/routes/chat/reorg.rs b/experimental/sgl-router/src/server/routes/chat/reorg.rs index 6911071c2f02..a71a4e9bde85 100644 --- a/experimental/sgl-router/src/server/routes/chat/reorg.rs +++ b/experimental/sgl-router/src/server/routes/chat/reorg.rs @@ -36,8 +36,13 @@ pub(super) async fn chat_completions( let resolver = resolvers .get(&model) .ok_or_else(|| ApiError::ModelNotFound(model.0.clone()))?; - // Length-based routing needs tokenization even for load-only group policies. - let request = PreparedChatRequest::prepare(ctx, model, fields, body, true)?; + // Tokenize only for what this model's buckets actually select on: a length + // bound, a context-capacity check, or a prefix-matching group policy. The + // same predicate gates `--no-tokenizer` at startup, so asking it here keeps + // the two in agreement and spares a load-only bucket a body parse whose + // result nothing reads. + let request = + PreparedChatRequest::prepare(ctx, model, fields, body, resolver.needs_request_tokens())?; let input_tokens = request.input_token_count as u64; let expected_peak_tokens = request .max_output_tokens diff --git a/experimental/sgl-router/src/server/routes/models.rs b/experimental/sgl-router/src/server/routes/models.rs index 3c1a063d6230..94379c69ead8 100644 --- a/experimental/sgl-router/src/server/routes/models.rs +++ b/experimental/sgl-router/src/server/routes/models.rs @@ -49,7 +49,7 @@ mod tests { let mut ctx = crate::server::app_context::AppContext::stub(); ctx.config.model = crate::config::ModelConfig { id: "qwen3".into(), - tokenizer_path: "x".into(), + tokenizer_path: Some("x".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/src/server/routes/tokenize.rs b/experimental/sgl-router/src/server/routes/tokenize.rs index 3f8528558700..0ee493a2dd28 100644 --- a/experimental/sgl-router/src/server/routes/tokenize.rs +++ b/experimental/sgl-router/src/server/routes/tokenize.rs @@ -6,6 +6,7 @@ use crate::server::error::ApiError; use crate::tokenizer::adapter; use axum::extract::State; use axum::Json; +use dynamo_tokenizers::Tokenizer; use serde::{Deserialize, Serialize}; use std::sync::Arc; @@ -40,14 +41,22 @@ pub struct DetokenizeResponse { pub text: String, } +/// Distinguish a served model started with `--tokenizer-path none` from an unknown one. +fn tokenizer_for(ctx: &AppContext, model: &str) -> Result, ApiError> { + ctx.tokenizers.get(model).ok_or_else(|| { + if model == ctx.config.model.id { + ApiError::BadRequest(format!("tokenizer disabled for model {model}")) + } else { + ApiError::ModelNotFound(model.to_owned()) + } + }) +} + pub async fn tokenize( State(ctx): State>, Json(req): Json, ) -> Result, ApiError> { - let tok = ctx - .tokenizers - .get(&req.model) - .ok_or_else(|| ApiError::ModelNotFound(req.model.clone()))?; + let tok = tokenizer_for(&ctx, &req.model)?; // Structured log on failure so an operator can correlate // "every encode for model X errors" against the route, model id, and // prompt size. The generic anyhow-chain log in ApiError::Internal still @@ -75,10 +84,7 @@ pub async fn detokenize( State(ctx): State>, Json(req): Json, ) -> Result, ApiError> { - let tok = ctx - .tokenizers - .get(&req.model) - .ok_or_else(|| ApiError::ModelNotFound(req.model.clone()))?; + let tok = tokenizer_for(&ctx, &req.model)?; let text = adapter::decode_complete(&tok, &req.tokens, req.skip_special_tokens).map_err(|e| { tracing::error!( @@ -117,7 +123,7 @@ mod tests { observability: Default::default(), model: crate::config::ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, @@ -344,6 +350,24 @@ mod tests { ); } + #[tokio::test] + async fn disabled_tokenizer_is_a_bad_request() { + let mut ctx = ctx_with_tiny(); + Arc::get_mut(&mut ctx).unwrap().tokenizers = Default::default(); + let res = crate::server::app::build_router(ctx) + .oneshot( + Request::builder() + .method("POST") + .uri("/v1/tokenize") + .header("content-type", "application/json") + .body(Body::from(r#"{"model":"tiny","prompt":"x"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(res.status(), StatusCode::BAD_REQUEST); + } + #[tokio::test] async fn unknown_model_404() { let app = crate::server::app::build_router(ctx_with_tiny()); diff --git a/experimental/sgl-router/src/tokenizer/mod.rs b/experimental/sgl-router/src/tokenizer/mod.rs index 89226dde9988..3a3b40dc61bb 100644 --- a/experimental/sgl-router/src/tokenizer/mod.rs +++ b/experimental/sgl-router/src/tokenizer/mod.rs @@ -78,13 +78,17 @@ impl TokenizerRegistry { pub fn load_from_config(cfg: &crate::config::Config) -> Result { let mut me = TokenizerRegistry::default(); let m = &cfg.model; - let (t, stats) = adapter::load_with(&m.tokenizer_path, m.tokenizer)?; + let Some(tokenizer_path) = &m.tokenizer_path else { + tracing::info!(model = %m.id, "tokenizer disabled; workers tokenize requests"); + return Ok(me); + }; + let (t, stats) = adapter::load_with(tokenizer_path, m.tokenizer)?; tracing::info!(model = %m.id, backend = stats.backend().as_str(), l1 = stats.l1_state().as_str(), l1_cache_mb = m.tokenizer.l1_cache_mb, "tokenizer loaded"); me.inner.insert(m.id.clone(), t); me.stats = stats; - match ChatFormatter::load(&m.id, &m.tokenizer_path) { + match ChatFormatter::load(&m.id, tokenizer_path) { Ok(Some(formatter)) => { let formatter = formatter.with_defaults(&m.default_chat_template_kwargs); me.formatters @@ -210,7 +214,7 @@ mod tests { observability: Default::default(), model: crate::config::ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, @@ -356,11 +360,20 @@ mod tests { #[test] fn missing_file_errors() { let mut c = cfg(); - c.model.tokenizer_path = "/nonexistent.json".into(); + c.model.tokenizer_path = Some("/nonexistent.json".into()); let err = TokenizerRegistry::load_from_config(&c).unwrap_err(); assert!(err.to_string().to_lowercase().contains("tokenizer")); } + #[test] + fn disabled_tokenizer_loads_an_empty_registry() { + let mut c = cfg(); + c.model.tokenizer_path = None; + let registry = TokenizerRegistry::load_from_config(&c).unwrap(); + assert!(registry.get("tiny").is_none()); + assert_eq!(registry.forwarding_scope("tiny"), ForwardingScope::Never); + } + #[test] fn model_files_json_reads_sibling() { let dir = tempfile::tempdir().unwrap(); @@ -420,7 +433,7 @@ mod tests { .unwrap(); std::fs::write(dir.path().join("chat_template.jinja"), "{% invalid %}").unwrap(); let mut cfg = cfg(); - cfg.model.tokenizer_path = tok.to_str().unwrap().to_owned(); + cfg.model.tokenizer_path = Some(tok.to_str().unwrap().to_owned()); let reg = TokenizerRegistry::load_from_config(&cfg).unwrap(); let tokenizer = reg.get(&cfg.model.id).unwrap(); diff --git a/experimental/sgl-router/src/workers/manager.rs b/experimental/sgl-router/src/workers/manager.rs index f43fabd99f2f..629a5e46f7b5 100644 --- a/experimental/sgl-router/src/workers/manager.rs +++ b/experimental/sgl-router/src/workers/manager.rs @@ -613,7 +613,7 @@ mod tests { observability: Default::default(), model: ModelConfig { id: id.into(), - tokenizer_path: "/tmp/x".into(), + tokenizer_path: Some("/tmp/x".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/component/discovery/static_urls.rs b/experimental/sgl-router/tests/component/discovery/static_urls.rs index 1b5490ad4455..5343da7ca582 100644 --- a/experimental/sgl-router/tests/component/discovery/static_urls.rs +++ b/experimental/sgl-router/tests/component/discovery/static_urls.rs @@ -130,7 +130,7 @@ async fn static_urls_pd_role_resolved_end_to_end() { observability: ObservabilityConfig::default(), model: sgl_router::config::ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: sgl_router::config::PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/component/tokenizer/render_parity.rs b/experimental/sgl-router/tests/component/tokenizer/render_parity.rs index bec05988af96..5c4bc6336a57 100644 --- a/experimental/sgl-router/tests/component/tokenizer/render_parity.rs +++ b/experimental/sgl-router/tests/component/tokenizer/render_parity.rs @@ -111,7 +111,7 @@ fn registry(model_id: &str, tokenizer_path: PathBuf) -> TokenizerRegistry { observability: ObservabilityConfig::default(), model: ModelConfig { id: model_id.into(), - tokenizer_path: tokenizer_path.to_str().unwrap().into(), + tokenizer_path: Some(tokenizer_path.to_str().unwrap().into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/proxy/bucket_routing.rs b/experimental/sgl-router/tests/proxy/bucket_routing.rs index 41e7fed6a0a3..a6de8a77f492 100644 --- a/experimental/sgl-router/tests/proxy/bucket_routing.rs +++ b/experimental/sgl-router/tests/proxy/bucket_routing.rs @@ -60,7 +60,7 @@ fn build_app_context( observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy, diff --git a/experimental/sgl-router/tests/proxy/chat_routing.rs b/experimental/sgl-router/tests/proxy/chat_routing.rs index 6b6ad3c40d7a..93a76b1bdeaa 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing.rs @@ -39,7 +39,7 @@ fn config_for(_worker_url: &str) -> Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs b/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs index b599d92ac4ce..1255558a945a 100644 --- a/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs +++ b/experimental/sgl-router/tests/proxy/chat_routing/reorg.rs @@ -781,3 +781,78 @@ async fn portless_prefill_is_not_dispatched_until_bootstrap_is_resolved() { assert_eq!(d["bootstrap_port"], 8997, "reorg={reorg}"); } } + +#[tokio::test] +async fn load_only_routing_forwards_messages_without_a_tokenizer() { + for (reorg, pd) in [(false, false), (false, true), (true, false), (true, true)] { + let prefill = MockWorker::start(vec![]).await; + let decode = MockWorker::start(vec![]).await; + let workers: &[_] = if pd { + &[ + ("p", Stage::Prefill, &prefill), + ("d", Stage::Decode, &decode), + ] + } else { + &[("d", Stage::Plain, &decode)] + }; + let mut ctx = context(workers, vec![]); + let mutable = Arc::get_mut(&mut ctx).unwrap(); + mutable.config.model.policy = PolicyKind::PowerOfTwo; + mutable.config.model.tokenizer_path = None; + mutable.tokenizers = + Arc::new(TokenizerRegistry::load_from_config(&mutable.config).unwrap()); + if reorg { + let state = sgl_router::state::kv_events::KvEventIndex::new(); + let (resolver, _) = sgl_router::policies_reorg::factory::build_resolver( + &mutable.config.model, + &state, + None, + ) + .unwrap(); + mutable.chat_routing = ChatRouting::Reorg([(ModelId("tiny".into()), resolver)].into()); + } else { + mutable.chat_routing = ChatRouting::Legacy; + mutable.policies = Arc::new(build_policy_registry(&mutable.config).unwrap()); + } + let response = build_router(ctx) + .oneshot(request(body("hello"))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "reorg={reorg} pd={pd}"); + let forwarded: serde_json::Value = + serde_json::from_slice(decode.captured.lock().unwrap().last_body.as_ref().unwrap()) + .unwrap(); + assert_eq!(forwarded["messages"], body("hello")["messages"]); + assert!( + forwarded.get("input_ids").is_none(), + "reorg={reorg} pd={pd}" + ); + } +} + +#[tokio::test] +async fn only_prefix_reading_routing_needs_request_tokens() { + for (policy, needs) in [ + (PolicyKind::PowerOfTwo, false), + (PolicyKind::SessionAware, false), + (PolicyKind::CacheAware, true), + ] { + let mut cfg = config_for(""); + cfg.model.policy = policy; + let legacy = build_policy_registry(&cfg).unwrap(); + assert_eq!( + ChatRouting::Legacy.needs_request_tokens(&legacy), + needs, + "legacy {policy:?}" + ); + let state = sgl_router::state::kv_events::KvEventIndex::new(); + let (resolver, _) = + sgl_router::policies_reorg::factory::build_resolver(&cfg.model, &state, None).unwrap(); + let reorg = ChatRouting::Reorg([(ModelId("tiny".into()), resolver)].into()); + assert_eq!( + reorg.needs_request_tokens(&PolicyRegistry::default()), + needs, + "reorg {policy:?}" + ); + } +} diff --git a/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs b/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs index 287106ea0f95..6deecf99c51d 100644 --- a/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs +++ b/experimental/sgl-router/tests/proxy/common/cache_aware_fixture.rs @@ -25,7 +25,7 @@ pub fn config() -> Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: MODEL.into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::CacheAware, diff --git a/experimental/sgl-router/tests/proxy/failover.rs b/experimental/sgl-router/tests/proxy/failover.rs index 8adb7dcd5223..f02fb321f335 100644 --- a/experimental/sgl-router/tests/proxy/failover.rs +++ b/experimental/sgl-router/tests/proxy/failover.rs @@ -34,7 +34,7 @@ async fn failover_when_one_worker_dies() { observability: Default::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/proxy/graceful_shutdown.rs b/experimental/sgl-router/tests/proxy/graceful_shutdown.rs index 67644ec0dfe9..e0dbb58580cd 100644 --- a/experimental/sgl-router/tests/proxy/graceful_shutdown.rs +++ b/experimental/sgl-router/tests/proxy/graceful_shutdown.rs @@ -52,7 +52,7 @@ fn build_ctx_with_worker(worker_url: &str) -> Arc { observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/proxy/header_forwarding.rs b/experimental/sgl-router/tests/proxy/header_forwarding.rs index c54a19428f45..d6c11cf5da7a 100644 --- a/experimental/sgl-router/tests/proxy/header_forwarding.rs +++ b/experimental/sgl-router/tests/proxy/header_forwarding.rs @@ -30,7 +30,7 @@ async fn forwards_whitelisted_headers_strips_others() { observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs b/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs index 46a7081529c5..082f50620a7d 100644 --- a/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs +++ b/experimental/sgl-router/tests/proxy/pd_bootstrap_injection.rs @@ -45,7 +45,7 @@ fn config() -> Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs index 97c2960fd55c..57b090369de7 100644 --- a/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs +++ b/experimental/sgl-router/tests/proxy/pd_pool_isolation.rs @@ -44,7 +44,7 @@ fn config() -> Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs b/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs index f11533af8552..fff13dc84530 100644 --- a/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs +++ b/experimental/sgl-router/tests/proxy/pd_protocol_binding.rs @@ -132,7 +132,7 @@ fn config() -> Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, diff --git a/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs b/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs index 29dc7719bf36..79440b116330 100644 --- a/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs +++ b/experimental/sgl-router/tests/proxy/roundrobin_input_ids.rs @@ -42,7 +42,7 @@ fn config() -> Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: MODEL.into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin, @@ -97,7 +97,7 @@ fn template_config(tokenizer_config: Value) -> (tempfile::TempDir, Config) { ) .unwrap(); let mut cfg = config(); - cfg.model.tokenizer_path = tokenizer.to_str().unwrap().into(); + cfg.model.tokenizer_path = Some(tokenizer.to_str().unwrap().into()); (dir, cfg) } @@ -436,7 +436,7 @@ async fn kimi_ids_forward_with_engine_rendering_fallback() { let fixture = kimi_fixture::tokenizer(); let mut cfg = config(); let path = fixture.path().join("tiktoken.model"); - cfg.model.tokenizer_path = path.display().to_string(); + cfg.model.tokenizer_path = Some(path.display().to_string()); let ctx = build_ctx_with_config(mock.url.clone(), cfg); for (content, kwargs) in [ ("literal <|open|> text", None), diff --git a/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs b/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs index 668caa625abf..21a450b10cc8 100644 --- a/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs +++ b/experimental/sgl-router/tests/proxy/shared_prefill_admission.rs @@ -148,7 +148,7 @@ fn config(policy: PolicyKind) -> Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy, diff --git a/experimental/sgl-router/tests/proxy/sticky_input_ids.rs b/experimental/sgl-router/tests/proxy/sticky_input_ids.rs index e7f1ba473774..2834c5c3d25c 100644 --- a/experimental/sgl-router/tests/proxy/sticky_input_ids.rs +++ b/experimental/sgl-router/tests/proxy/sticky_input_ids.rs @@ -53,7 +53,7 @@ fn config() -> Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: MODEL.into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::Sticky, diff --git a/experimental/sgl-router/tests/proxy/sticky_routing.rs b/experimental/sgl-router/tests/proxy/sticky_routing.rs index 070e328fc74b..f2b19a2a38e8 100644 --- a/experimental/sgl-router/tests/proxy/sticky_routing.rs +++ b/experimental/sgl-router/tests/proxy/sticky_routing.rs @@ -41,7 +41,7 @@ fn build_sticky_ctx(header_name: &str, worker_urls: &[String]) -> Arc Config { observability: ObservabilityConfig::default(), model: ModelConfig { id: "tiny".into(), - tokenizer_path: "tests/fixtures/tiny_tokenizer.json".into(), + tokenizer_path: Some("tests/fixtures/tiny_tokenizer.json".into()), disable_input_ids_forwarding: false, tokenizer: Default::default(), policy: PolicyKind::RoundRobin,