[sgl-router] Allow load-only routing without a tokenizer

experimental/sgl-router/README.md

@@ -88,6 +88,14 @@ buckets and are not accepted on this path.
 
 Omitting `--chat-routing` keeps the existing policies and defaults.
 
+### Optional tokenizer for load-only routing
+
+`--no-tokenizer` skips tokenizer loading for load-only policies such as
+`power_of_two` and `session_aware`, on either routing path. Workers tokenize the
+original messages, and `/v1/tokenize` and `/v1/detokenize` are unavailable.
+Cache-aware routing, prefix-cache terms or filters, and `--bucket-config` still
+require a tokenizer.
+
 ### Fleet-wide sampling contract
 
 `--override-sampling-params` fixes the sampling configuration for every client

experimental/sgl-router/src/buckets_reorg.rs

@@ -164,6 +164,21 @@ impl Bucket {
         Ok(())
     }
 
+    /// Whether choosing or serving this bucket reads request tokens: a length
+    /// bound, or a group policy that matches prefixes.
+    pub fn needs_request_tokens(&self) -> bool {
+        let groups: &[&EngineGroup] = match &self.groups {
+            BucketGroups::Plain(group) => &[group],
+            BucketGroups::Pd { prefill, decode } => &[prefill, decode],
+        };
+        self.limits.min.is_some()
+            || self.limits.max.is_some()
+            || self.max_context_tokens.is_some()
+            || groups
+                .iter()
+                .any(|group| group.policy.needs_request_tokens())
+    }
+
     /// Select this bucket's plain engine or complete P/D pair, without dispatching.
     /// A failed group reports its stage; the caller may then try another bucket.
     pub async fn pick_engines(
@@ -268,6 +283,11 @@ impl BucketResolver {
         })
     }
 
+    /// Whether any bucket reads request tokens; startup rejects this under `--no-tokenizer`.
+    pub fn needs_request_tokens(&self) -> bool {
+        self.buckets.iter().any(Bucket::needs_request_tokens)
+    }
+
     /// Return all length-compatible buckets, ordered by unmet SLO preferences,
     /// then input capacity, rank, and ID. Both preferences have equal weight.
     /// The caller tries their groups in order until a complete engine selection succeeds.
@@ -317,3 +337,59 @@ impl BucketResolver {
         Ok(buckets)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::policies_reorg::power_of_two::PowerOfTwoPolicy;
+    use crate::state::load_monitor::engine_reported_load::EngineReportedLoadTable;
+
+    /// A bucket whose group policy reads no tokens, so only the bucket's own
+    /// length fields can make it token-hungry.
+    fn load_only_bucket(id: &str) -> Bucket {
+        let policy: Arc<dyn Policy> =
+            Arc::new(PowerOfTwoPolicy::new(EngineReportedLoadTable::new()));
+        Bucket::new(id, BucketGroups::Plain(EngineGroup::new(policy)))
+    }
+
+    /// `reorg::chat_completions` gates request tokenization on this predicate, so
+    /// every length field [`Bucket::fits`] reads has to keep it true. Drop one and
+    /// a length-bounded bucket would silently select on the request-size estimate
+    /// that [`crate::server::routes::chat`] falls back to without tokens.
+    #[test]
+    fn a_bucket_needs_tokens_for_every_length_field_it_selects_on() {
+        type Mutate = fn(&mut Bucket);
+        assert!(
+            !load_only_bucket("b").needs_request_tokens(),
+            "a load-only bucket selects on nothing that needs tokens"
+        );
+        for (field, mutate) in [
+            (
+                "limits.min",
+                (|b: &mut Bucket| b.limits.min = Some(1)) as Mutate,
+            ),
+            ("limits.max", |b: &mut Bucket| b.limits.max = Some(1)),
+            ("max_context_tokens", |b: &mut Bucket| {
+                b.max_context_tokens = Some(1)
+            }),
+        ] {
+            let mut bucket = load_only_bucket("b");
+            mutate(&mut bucket);
+            assert!(bucket.needs_request_tokens(), "{field}");
+        }
+    }
+
+    /// One bounded bucket makes every request to the model tokenize: the bucket is
+    /// chosen from the token count, so it cannot be known to be irrelevant first.
+    #[test]
+    fn a_resolver_needs_tokens_when_any_bucket_does() {
+        let mut bounded = load_only_bucket("bounded");
+        bounded.limits.max = Some(1);
+        assert!(!BucketResolver::new(vec![load_only_bucket("a")])
+            .unwrap()
+            .needs_request_tokens());
+        assert!(BucketResolver::new(vec![load_only_bucket("a"), bounded])
+            .unwrap()
+            .needs_request_tokens());
+    }
+}

experimental/sgl-router/src/config/cli.rs

@@ -67,6 +67,11 @@ pub struct ModelArgs {
     #[arg(long)]
     pub tokenizer_path: Option<String>,
 
+    /// Skip loading a tokenizer; workers tokenize requests. Load-only policies only:
+    /// cache-aware routing, prefix-cache terms or filters, and length buckets require one.
+    #[arg(long, conflicts_with = "tokenizer_path")]
+    pub no_tokenizer: bool,
+
     /// Disable generated input_ids; workers tokenize messages, while routing still renders locally.
     /// Use for worker parser/template overrides or template stop strings.
     #[arg(long)]
@@ -473,10 +478,11 @@ impl Cli {
                 log_format: self.server.log_format,
             },
             model: ModelConfig {
-                tokenizer_path: self
-                    .model
-                    .tokenizer_path
-                    .unwrap_or_else(|| self.model.model_id.clone()),
+                tokenizer_path: (!self.model.no_tokenizer).then(|| {
+                    self.model
+                        .tokenizer_path
+                        .unwrap_or_else(|| self.model.model_id.clone())
+                }),
                 id: self.model.model_id,
                 disable_input_ids_forwarding: self.model.disable_input_ids_forwarding,
                 tokenizer: TokenizerConfig {
@@ -1195,7 +1201,7 @@ mod tests {
         ])
         .unwrap();
         assert_eq!(c.model.id, "Qwen/Qwen3-0.6B");
-        assert_eq!(c.model.tokenizer_path, "Qwen/Qwen3-0.6B");
+        assert_eq!(c.model.tokenizer_path.as_deref(), Some("Qwen/Qwen3-0.6B"));
     }
 
     #[test]
@@ -1209,7 +1215,25 @@ mod tests {
             "http://x:30000",
         ])
         .unwrap();
-        assert_eq!(c.model.tokenizer_path, "/models/qwen3/tokenizer.json");
+        assert_eq!(
+            c.model.tokenizer_path.as_deref(),
+            Some("/models/qwen3/tokenizer.json")
+        );
+    }
+
+    #[test]
+    fn no_tokenizer_disables_loading_and_conflicts_with_a_path() {
+        let base = ["--model-id", "qwen3", "--worker-urls", "http://x:30000"];
+        let c = into_config(&[&base[..], &["--no-tokenizer"]].concat()).unwrap();
+        assert_eq!(c.model.tokenizer_path, None);
+        assert!(into_config(
+            &[
+                &base[..],
+                &["--no-tokenizer", "--tokenizer-path", "/t.json"]
+            ]
+            .concat()
+        )
+        .is_err());
     }
 
     #[test]

experimental/sgl-router/src/config/mod.rs

@@ -44,6 +44,11 @@ impl Config {
     /// Validate invariants not enforced by the CLI parser.
     pub(crate) fn validate(&self) -> Result<()> {
         ensure!(!self.model.id.is_empty(), "model id must be non-empty");
+        // Policies that read the prompt are checked against the built routing at startup.
+        ensure!(
+            self.model.tokenizer_path.is_some() || self.model.bucket_config.is_none(),
+            "--no-tokenizer is incompatible with --bucket-config"
+        );
         if let Some(bucket_config) = self.model.bucket_config.as_ref() {
             validate_bucket_config(bucket_config)?;
         }
@@ -243,7 +248,7 @@ mod tests {
             observability: ObservabilityConfig::default(),
             model: ModelConfig {
                 id: model_id.into(),
-                tokenizer_path: "/tmp/tok.json".into(),
+                tokenizer_path: Some("/tmp/tok.json".into()),
                 disable_input_ids_forwarding: false,
                 tokenizer: Default::default(),
                 policy: PolicyKind::RoundRobin,
@@ -271,6 +276,20 @@ mod tests {
         cfg("qwen3", &["http://10.0.0.1:30000"]).validate().unwrap();
     }
 
+    #[test]
+    fn disabled_tokenizer_rejects_length_buckets() {
+        let mut c = cfg("qwen3", &["http://10.0.0.1:30000"]);
+        c.model.tokenizer_path = None;
+        c.validate().unwrap();
+        c.model.bucket_config = Some(BucketConfig {
+            buckets: vec![],
+            ttft_slo_policy: SloBucketPolicy::Disabled,
+            tps_slo_policy: SloBucketPolicy::Disabled,
+        });
+        let err = c.validate().unwrap_err().to_string();
+        assert!(err.contains("--no-tokenizer"), "{err}");
+    }
+
     #[test]
     fn rejects_empty_model_id() {
         let err = cfg("", &["http://10.0.0.1:30000"])

experimental/sgl-router/src/config/types.rs

@@ -327,8 +327,8 @@ impl Default for ObservabilityConfig {
 pub struct ModelConfig {
     pub id: String,
     /// Local tokenizer.json or HuggingFace repo id; defaults to `id`.
-    /// Resolved by [`crate::tokenizer::adapter::load`].
-    pub tokenizer_path: String,
+    /// Resolved by [`crate::tokenizer::adapter::load`]; `None` (`--no-tokenizer`) disables it.
+    pub tokenizer_path: Option<String>,
     /// Disable router-generated input IDs for this model; keep routing tokenization.
     /// Use when workers have rendering defaults or template stops the router cannot see.
     pub disable_input_ids_forwarding: bool,

experimental/sgl-router/src/main.rs

@@ -1,7 +1,7 @@
 // SPDX-FileCopyrightText: Copyright (c) 2026 The SGLang Authors
 // SPDX-License-Identifier: Apache-2.0
 
-use anyhow::{Context, Result};
+use anyhow::{ensure, Context, Result};
 use clap::Parser;
 use sgl_kv_indexer::{GrpcPrefixIndex, PrefixIndex, PrefixIndexConfig};
 use sgl_router::{
@@ -107,6 +107,13 @@ async fn main() -> Result<()> {
         }
     };
 
+    // Ask the built policies, so any term or filter that reads the prompt is covered.
+    ensure!(
+        config.model.tokenizer_path.is_some()
+            || !chat_routing.needs_request_tokens(&routing_policies),
+        "--no-tokenizer is incompatible with cache-aware routing and prefix-cache terms or filters"
+    );
+
     // Track this router's local view of in-flight requests.
     let (local_inflight_requests, inflight_cleanup) = start_local_inflight_tracker(&config);
 

experimental/sgl-router/src/policies/factory.rs

@@ -336,7 +336,7 @@ mod tests {
             observability: Default::default(),
             model: ModelConfig {
                 id: id.into(),
-                tokenizer_path: "/tmp/x".into(),
+                tokenizer_path: Some("/tmp/x".into()),
                 disable_input_ids_forwarding: false,
                 tokenizer: Default::default(),
                 policy,
@@ -359,6 +359,44 @@ mod tests {
         }
     }
 
+    #[test]
+    fn only_prefix_reading_policies_need_request_tokens() {
+        fn fuse(model: &mut ModelConfig, kinds: &[ScoreTermKind]) {
+            model.policy = PolicyKind::FusedScore;
+            model.fused = Some(
+                (kinds.iter())
+                    .map(|&kind| crate::config::FusedTerm { kind, weight: None })
+                    .collect(),
+            );
+        }
+        type Mutate = fn(&mut ModelConfig);
+        let cases: [(Mutate, bool); 8] = [
+            (|_| {}, false),
+            (|m| m.policy = PolicyKind::PowerOfTwo, false),
+            (|m| m.policy = PolicyKind::SessionAware, false),
+            (|m| m.policy = PolicyKind::CacheAware, true),
+            (|m| fuse(m, &crate::config::DEFAULT_FUSE), true),
+            (|m| fuse(m, &[ScoreTermKind::LoadBased]), false),
+            (|m| fuse(m, &[ScoreTermKind::PrefixCache]), true),
+            (
+                |m| {
+                    m.eligibility = Some(EligibilityConfig {
+                        filters: vec![FilterKind::PrefixCache],
+                        min_prefix_share: Some(0.6),
+                        ..Default::default()
+                    })
+                },
+                true,
+            ),
+        ];
+        for (i, (mutate, needs)) in cases.into_iter().enumerate() {
+            let mut cfg = cfg_with_model("modelA", PolicyKind::RoundRobin);
+            mutate(&mut cfg.model);
+            let registry = build_registry_with_defaults(&cfg).unwrap();
+            assert_eq!(registry.needs_request_tokens(), needs, "case {i}");
+        }
+    }
+
     #[test]
     fn build_policy_kind_only_covers_all_variants() {
         for (kind, needs_load_snapshot, needs_dispatch_timestamps) in [

experimental/sgl-router/src/policies/mod.rs

@@ -572,6 +572,13 @@ impl PolicyRegistry {
         self.by_model.get(model).map(|p| p.clone())
     }
 
+    /// Whether any registered policy reads request tokens; startup rejects this under `--no-tokenizer`.
+    pub fn needs_request_tokens(&self) -> bool {
+        self.by_model
+            .iter()
+            .any(|entry| entry.value().needs_request_tokens())
+    }
+
     /// Attaches metrics to each registered policy.
     pub fn attach_metrics(&self, metrics: Arc<MetricsRegistry>) {
         for entry in self.by_model.iter() {

experimental/sgl-router/src/policies_reorg/cache_aware.rs

@@ -401,6 +401,10 @@ impl Policy for CacheAwarePolicy {
         stage != Stage::Decode
     }
 
+    fn needs_request_tokens(&self) -> bool {
+        true
+    }
+
     fn pick<'a>(
         &'a self,
         engines: &'a [Arc<Worker>],

experimental/sgl-router/src/policies_reorg/mod.rs

@@ -96,6 +96,12 @@ pub trait Policy: Send + Sync + Debug {
         true
     }
 
+    /// Whether picks read request tokens (prefix matching). Load-only policies
+    /// work from the request-size estimate when no tokenizer is loaded.
+    fn needs_request_tokens(&self) -> bool {
+        false
+    }
+
     /// Runs on a miss within the same candidates; never on an admission rejection.
     fn fallback(&self) -> Option<&dyn Policy> {
         None

experimental/sgl-router/src/server/app_context.rs

@@ -38,6 +38,16 @@ pub enum ChatRouting {
     Reorg(HashMap<ModelId, BucketResolver>),
 }
 
+impl ChatRouting {
+    /// Whether the built routing reads request tokens; `legacy` holds the legacy policies.
+    pub fn needs_request_tokens(&self, legacy: &PolicyRegistry) -> bool {
+        match self {
+            Self::Legacy => legacy.needs_request_tokens(),
+            Self::Reorg(resolvers) => resolvers.values().any(BucketResolver::needs_request_tokens),
+        }
+    }
+}
+
 pub struct AppContext {
     pub config: Config,
     pub tokenizers: Arc<TokenizerRegistry>,
@@ -197,7 +207,7 @@ impl AppContext {
                 observability: Default::default(),
                 model: crate::config::ModelConfig {
                     id: "stub-model".into(),
-                    tokenizer_path: "stub".into(),
+                    tokenizer_path: Some("stub".into()),
                     disable_input_ids_forwarding: false,
                     tokenizer: Default::default(),
                     policy: crate::config::PolicyKind::RoundRobin,

experimental/sgl-router/src/server/routes/chat/reorg.rs

@@ -36,8 +36,13 @@ pub(super) async fn chat_completions(
     let resolver = resolvers
         .get(&model)
         .ok_or_else(|| ApiError::ModelNotFound(model.0.clone()))?;
-    // Length-based routing needs tokenization even for load-only group policies.
-    let request = PreparedChatRequest::prepare(ctx, model, fields, body, true)?;
+    // Tokenize only for what this model's buckets actually select on: a length
+    // bound, a context-capacity check, or a prefix-matching group policy. The
+    // same predicate gates `--no-tokenizer` at startup, so asking it here keeps
+    // the two in agreement and spares a load-only bucket a body parse whose
+    // result nothing reads.
+    let request =
+        PreparedChatRequest::prepare(ctx, model, fields, body, resolver.needs_request_tokens())?;
     let input_tokens = request.input_token_count as u64;
     let expected_peak_tokens = request
         .max_output_tokens

experimental/sgl-router/src/server/routes/models.rs

@@ -49,7 +49,7 @@ mod tests {
         let mut ctx = crate::server::app_context::AppContext::stub();
         ctx.config.model = crate::config::ModelConfig {
             id: "qwen3".into(),
-            tokenizer_path: "x".into(),
+            tokenizer_path: Some("x".into()),
             disable_input_ids_forwarding: false,
             tokenizer: Default::default(),
             policy: PolicyKind::RoundRobin,