feat(frontend): add basetenkenizer backend

components/src/dynamo/frontend/frontend_args.py

@@ -88,7 +88,7 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AicPerfConfigBase):
     trust_remote_code: bool
     frontend_route_extensions: list[str]
 
-    _VALID_TOKENIZER_BACKENDS = {"default", "fastokens"}
+    _VALID_TOKENIZER_BACKENDS = {"default", "fastokens", "basetenkenizer"}
 
     def validate(self) -> None:
         if self.load_aware:
@@ -505,11 +505,12 @@ def add_arguments(self, parser) -> None:
             default="default",
             dest="tokenizer_backend",
             help=(
-                "Tokenizer backend for BPE models: 'default' (HuggingFace tokenizers library) "
-                "or 'fastokens' (fastokens crate for high-performance BPE encoding). "
-                "Decoding always uses HuggingFace. Has no effect on TikToken models."
+                "Tokenizer backend for BPE models: 'default' (HuggingFace tokenizers library), "
+                "'fastokens' (fastokens crate for high-performance BPE encoding), or "
+                "'basetenkenizer' (Baseten Tokenizer for native encoding and decoding). "
+                "Has no effect on TikToken models."
             ),
-            choices=["default", "fastokens"],
+            choices=["default", "fastokens", "basetenkenizer"],
         )
 
         add_negatable_bool_argument(

docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md

@@ -2,10 +2,10 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 title: Tokenizer
-subtitle: Selects between the default HuggingFace and fastokens tokenizer backends for BPE models served through the Dynamo Frontend.
+subtitle: Selects the HuggingFace, fastokens, or Baseten tokenizer backend for BPE models served through the Dynamo Frontend.
 ---
 
-The Dynamo Frontend supports multiple tokenizer backends for BPE-based `tokenizer.json` models. `BPE` is the underlying tokenization algorithm, not a backend-specific feature: both the default HuggingFace path and the `fastokens` path can serve these models. The backend choice controls which implementation performs tokenization before requests are sent to the inference engine.
+The Dynamo Frontend supports multiple tokenizer backends for BPE-based `tokenizer.json` models. `BPE` is the underlying tokenization algorithm, not a backend-specific feature: the default HuggingFace, `fastokens`, and `basetenkenizer` paths can all serve supported BPE models. The backend choice controls which implementation performs tokenization before requests are sent to the inference engine.
 
 ## Tokenizer Backends
 
@@ -21,10 +21,17 @@ It is a _hybrid_ backend: encoding uses `fastokens` while decoding falls back to
 
 Use this backend when tokenization is a measurable bottleneck, for example on high-concurrency prefill-heavy workloads.
 
+#### `basetenkenizer` Native Encoder and Decoder
+
+The `basetenkenizer` backend uses the Baseten Tokenizer implementation exposed by `dynamo-tokenizers`, a high-performance Rust BPE implementation for inference. Unlike the hybrid `fastokens` path, it performs both encoding and decoding natively and supports segmented encoding for renderers that must preserve trusted control-token boundaries.
+
+Use this backend for supported `tokenizer.json` models when you need Baseten Tokenizer behavior, including token-compatible Kimi tokenizer artifacts.
+
 #### Compatibility notes:
 
 - Works with standard BPE `tokenizer.json` files (Qwen, LLaMA, GPT-family, Mistral, DeepSeek, etc.).
-- If `fastokens` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace; requests are never dropped.
+- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace; requests are never dropped.
+- Special tokens declared only in a sibling `tokenizer_config.json` are preserved for Baseten encoding and decoding and for Dynamo's L1 prefix-cache boundaries.
 - Has no effect on TikToken-format tokenizers (`.model` / `.tiktoken` files), which always use the TikToken backend.
 
 ## Configuration
@@ -33,7 +40,7 @@ Set the backend with a CLI flag or environment variable. The CLI flag takes prec
 
 | CLI Argument | Env Var | Valid values | Default |
 |---|---|---|---|
-| `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens` | `default` |
+| `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens`, `basetenkenizer` | `default` |
 
 **Examples:**
 
@@ -44,13 +51,17 @@ python -m dynamo.frontend --tokenizer fastokens
 # Environment variable
 export DYN_TOKENIZER=fastokens
 python -m dynamo.frontend
+
+# Baseten Tokenizer
+python -m dynamo.frontend --tokenizer basetenkenizer
 ```
 
 ## Dynamo Frontend Behavior
 
-When `DYN_TOKENIZER=fastokens` is set:
+When a non-default backend is selected:
 
-1. The frontend passes the environment variable to the Rust runtime.
-2. When building the tokenizer for a model, `ModelDeploymentCard::tokenizer()` attempts to load `fastokens::Tokenizer` from the same `tokenizer.json` file.
-3. If loading succeeds, a hybrid `FastTokenizer` is created that encodes with `fastokens` and decodes with HuggingFace.
-4. If loading fails (unsupported tokenizer features, missing file, etc.), the frontend logs a warning and falls back to the standard HuggingFace backend; no operator intervention is needed.
+1. The frontend resolves `--tokenizer` / `DYN_TOKENIZER` and passes the selected backend to the Rust runtime.
+2. `ModelDeploymentCard::tokenizer()` loads the HuggingFace tokenizer first for fallback behavior and L1 cache special-token metadata.
+3. Dynamo constructs `FastTokenizer` for `fastokens` or `BasetenTokenizer` for `basetenkenizer` from the same `tokenizer.json` file.
+4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace.
+5. When the L1 prefix cache is enabled, Dynamo wraps the selected backend with the same special-token boundary metadata and cache metrics used by the default path.

docs/fern/pages/reference/components/frontend-configuration.mdx

@@ -497,9 +497,9 @@ See the [Frontend Guide](../../developer-guide/knowledge-base/modular-components
 ## Tokenizer
 
 <ParamField path="--tokenizer" type="string" default="default">
-  Tokenizer implementation. `default` uses HuggingFace; `fastokens` uses the high-performance Rust tokenizer. See [Tokenizer](../../developer-guide/knowledge-base/modular-components/frontend/tokenizer.md).
+  Tokenizer implementation. `default` uses HuggingFace; `fastokens` uses the high-performance hybrid encoder; and `basetenkenizer` uses Baseten Tokenizer for native encoding and decoding. See [Tokenizer](../../developer-guide/knowledge-base/modular-components/frontend/tokenizer.md).
 
-  <span className="enum-values"><span className="enum-label">Allowed values:</span> <Badge intent="note" minimal>default</Badge> <Badge intent="note" minimal>fastokens</Badge></span>
+  <span className="enum-values"><span className="enum-label">Allowed values:</span> <Badge intent="note" minimal>default</Badge> <Badge intent="note" minimal>fastokens</Badge> <Badge intent="note" minimal>basetenkenizer</Badge></span>
 
   Environment variable: `DYN_TOKENIZER`
 </ParamField>

docs/fern/pages/use-cases/fastokens-tokenizer/overview.md

@@ -156,11 +156,11 @@ For any new model, validate on representative prompts before rolling out broadly
   </Accordion>
 
   <Accordion title="Why do the logs show an unrecognized DYN_TOKENIZER value?">
-    Use only `fastokens` or `default` for `DYN_TOKENIZER`. Values such as `fast`, `hf`, or `huggingface` are benchmark-runner aliases, not valid values for the frontend environment variable.
+    Use `default`, `fastokens`, or `basetenkenizer` for `DYN_TOKENIZER`. Values such as `fast`, `hf`, or `huggingface` are benchmark-runner aliases, not valid values for the frontend environment variable.
   </Accordion>
 
   <Accordion title="What happens when the model uses .model or .tiktoken files?">
-    The `fastokens` setting has no effect for TikToken-format tokenizers. Dynamo uses the existing TikToken backend, so you should not expect the `Using fastokens tokenizer backend` log or a `fastokens` speedup.
+    The `fastokens` and `basetenkenizer` settings have no effect for TikToken-format tokenizers. Dynamo uses the existing TikToken backend, so you should not expect either alternate-backend activation log or speedup.
   </Accordion>
 
   <Accordion title="Why doesn't TTFT improve?">

lib/llm/src/local_model/runtime_config.rs

@@ -62,13 +62,15 @@ pub const VLLM_INFERENCE_V1_GENERATE_CAPABILITY: &str = "vllm_inference_v1_gener
 pub enum TokenizerBackend {
     Default,
     Fastokens,
+    Basetenkenizer,
 }
 
 impl TokenizerBackend {
     pub fn as_str(self) -> &'static str {
         match self {
             Self::Default => "default",
             Self::Fastokens => "fastokens",
+            Self::Basetenkenizer => "basetenkenizer",
         }
     }
 
@@ -79,11 +81,12 @@ impl TokenizerBackend {
     pub fn from_env_or_default() -> Self {
         match std::env::var(ENV_TOKENIZER_BACKEND) {
             Ok(v) if v == "fastokens" => Self::Fastokens,
+            Ok(v) if v == "basetenkenizer" => Self::Basetenkenizer,
             Ok(v) if v == "default" || v.is_empty() => Self::Default,
             Ok(v) => {
                 tracing::warn!(
                     value = %v,
-                    "Unrecognized DYN_TOKENIZER value, expected 'fastokens' or 'default'; falling back to default"
+                    "Unrecognized DYN_TOKENIZER value, expected 'default', 'fastokens', or 'basetenkenizer'; falling back to default"
                 );
                 Self::Default
             }
@@ -99,8 +102,9 @@ impl FromStr for TokenizerBackend {
         match value {
             "default" => Ok(Self::Default),
             "fastokens" => Ok(Self::Fastokens),
+            "basetenkenizer" => Ok(Self::Basetenkenizer),
             _ => Err(format!(
-                "invalid tokenizer backend '{value}' (expected 'default' or 'fastokens')"
+                "invalid tokenizer backend '{value}' (expected 'default', 'fastokens', or 'basetenkenizer')"
             )),
         }
     }
@@ -683,6 +687,14 @@ mod tests {
             );
         });
 
+        temp_env::with_vars([(ENV_TOKENIZER_BACKEND, Some("basetenkenizer"))], || {
+            let cfg = ModelRuntimeConfig::default();
+            assert_eq!(
+                cfg.effective_tokenizer_backend(),
+                TokenizerBackend::Basetenkenizer
+            );
+        });
+
         temp_env::with_vars([(ENV_TOKENIZER_BACKEND, Some("default"))], || {
             let cfg = ModelRuntimeConfig::default();
             assert_eq!(cfg.effective_tokenizer_backend(), TokenizerBackend::Default);
@@ -715,18 +727,49 @@ mod tests {
                 TokenizerBackend::Fastokens
             );
         });
+
+        temp_env::with_vars([(ENV_TOKENIZER_BACKEND, Some("fastokens"))], || {
+            let cfg = ModelRuntimeConfig {
+                tokenizer_backend: Some(TokenizerBackend::Basetenkenizer),
+                ..Default::default()
+            };
+            assert_eq!(
+                cfg.effective_tokenizer_backend(),
+                TokenizerBackend::Basetenkenizer
+            );
+        });
     }
 
     #[test]
     fn tokenizer_backend_roundtrips_through_serde_json() {
-        let cfg = ModelRuntimeConfig {
-            tokenizer_backend: Some(TokenizerBackend::Fastokens),
-            ..Default::default()
-        };
-        let json = serde_json::to_string(&cfg).unwrap();
-        assert!(json.contains("\"tokenizer_backend\":\"fastokens\""));
-        let parsed: ModelRuntimeConfig = serde_json::from_str(&json).unwrap();
-        assert_eq!(parsed.tokenizer_backend, Some(TokenizerBackend::Fastokens));
+        for backend in [
+            TokenizerBackend::Default,
+            TokenizerBackend::Fastokens,
+            TokenizerBackend::Basetenkenizer,
+        ] {
+            let cfg = ModelRuntimeConfig {
+                tokenizer_backend: Some(backend),
+                ..Default::default()
+            };
+            let json = serde_json::to_string(&cfg).unwrap();
+            assert!(json.contains(&format!("\"tokenizer_backend\":\"{}\"", backend.as_str())));
+            let parsed: ModelRuntimeConfig = serde_json::from_str(&json).unwrap();
+            assert_eq!(parsed.tokenizer_backend, Some(backend));
+        }
+    }
+
+    #[test]
+    fn tokenizer_backend_string_values_are_strict() {
+        for backend in [
+            TokenizerBackend::Default,
+            TokenizerBackend::Fastokens,
+            TokenizerBackend::Basetenkenizer,
+        ] {
+            assert_eq!(backend.as_str().parse(), Ok(backend));
+        }
+
+        let error = "baseten".parse::<TokenizerBackend>().unwrap_err();
+        assert!(error.contains("basetenkenizer"));
     }
 
     #[test]

lib/llm/src/model_card.rs

@@ -18,7 +18,7 @@ use std::sync::{Arc, OnceLock};
 
 use crate::common::checked_file::CheckedFile;
 use crate::entrypoint::RouterConfig;
-use crate::local_model::runtime_config::ModelRuntimeConfig;
+use crate::local_model::runtime_config::{ModelRuntimeConfig, TokenizerBackend};
 use crate::model_type::{ModelInput, ModelType};
 use crate::protocols::tensor::TensorModelConfig;
 use anyhow::{Context, Result};
@@ -1174,8 +1174,8 @@ impl ModelDeploymentCard {
     /// This supports both HuggingFace `tokenizer.json` and tiktoken `.model`/`.tiktoken` files.
     ///
     /// Tokenizer backend controls:
-    /// - `runtime_config.tokenizer_backend=fastokens` — use `fastokens` as the encoding backend
-    /// - `DYN_TOKENIZER=fastokens` — fallback backend for callers without explicit runtime config
+    /// - `runtime_config.tokenizer_backend` — select `default`, `fastokens`, or `basetenkenizer`
+    /// - `DYN_TOKENIZER` — fallback backend for callers without explicit runtime config
     /// - `DYN_TOKENIZER_CACHE=0` — disable the L1 prefix cache that records tokenizations
     ///   at special-token boundaries (enabled by default; any other value keeps it enabled)
     /// - `DYN_TOKENIZER_CACHE_BYTES=<n>` — L1 cache byte budget (default 64 MiB)
@@ -1185,10 +1185,7 @@ impl ModelDeploymentCard {
     ///   per-turn tokenization cost flat instead of growing with history. Set to `0` to
     ///   fall back to the original hit-without-insert behavior.
     pub fn tokenizer(&self) -> anyhow::Result<crate::tokenizers::Tokenizer> {
-        let use_fast = self
-            .runtime_config
-            .effective_tokenizer_backend()
-            .is_fastokens();
+        let tokenizer_backend = self.runtime_config.effective_tokenizer_backend();
 
         let cache_enabled =
             tokenizer_cache_enabled(std::env::var("DYN_TOKENIZER_CACHE").ok().as_deref());
@@ -1207,9 +1204,9 @@ impl ModelDeploymentCard {
                 })?;
 
                 // Load HF first — needed both for fallback and (if cache is on) for
-                // extracting special-token strings. `FastTokenizer` does not re-expose
-                // `get_added_tokens_decoder`, so we must capture specials from the raw
-                // HF tokenizer before any swap.
+                // extracting special-token strings. Alternate backends do not re-expose
+                // `get_added_tokens_decoder`, so capture specials from the raw HF
+                // tokenizer before any swap.
                 let mut hf = HfTokenizer::from_file(p)
                     .inspect_err(|err| {
                         if let Some(serde_err) = err.downcast_ref::<serde_json::Error>()
@@ -1260,30 +1257,54 @@ impl ModelDeploymentCard {
                     |hf: HfTokenizer| crate::tokenizers::HuggingFaceTokenizer::from_tokenizer(hf);
 
                 // Pick the inner backend.
-                let raw: Arc<dyn crate::tokenizers::traits::Tokenizer> = if use_fast {
-                    if let Some(path_str) = p.to_str() {
-                        match crate::tokenizers::FastTokenizer::from_file(path_str) {
-                            Ok(fast) => {
-                                tracing::info!("Using fastokens tokenizer backend");
-                                Arc::new(fast)
+                let raw: Arc<dyn crate::tokenizers::traits::Tokenizer> = match tokenizer_backend {
+                    TokenizerBackend::Default => Arc::new(wrap_hf(hf)),
+                    TokenizerBackend::Fastokens => {
+                        if let Some(path_str) = p.to_str() {
+                            match crate::tokenizers::FastTokenizer::from_file(path_str) {
+                                Ok(fast) => {
+                                    tracing::info!("Using fastokens tokenizer backend");
+                                    Arc::new(fast)
+                                }
+                                Err(e) => {
+                                    tracing::warn!(
+                                        %e,
+                                        "Failed to load fastokens, falling back to HuggingFace"
+                                    );
+                                    Arc::new(wrap_hf(hf))
+                                }
                             }
-                            Err(e) => {
-                                tracing::warn!(
-                                    %e,
-                                    "Failed to load fastokens, falling back to HuggingFace"
-                                );
-                                Arc::new(wrap_hf(hf))
+                        } else {
+                            tracing::warn!(
+                                path = %p.display(),
+                                "Tokenizer path contains non-UTF-8 characters, skipping fastokens; falling back to HuggingFace"
+                            );
+                            Arc::new(wrap_hf(hf))
+                        }
+                    }
+                    TokenizerBackend::Basetenkenizer => {
+                        if let Some(path_str) = p.to_str() {
+                            match crate::tokenizers::BasetenTokenizer::from_file(path_str) {
+                                Ok(baseten) => {
+                                    tracing::info!("Using basetenkenizer tokenizer backend");
+                                    Arc::new(baseten)
+                                }
+                                Err(e) => {
+                                    tracing::warn!(
+                                        %e,
+                                        "Failed to load basetenkenizer, falling back to HuggingFace"
+                                    );
+                                    Arc::new(wrap_hf(hf))
+                                }
                             }
+                        } else {
+                            tracing::warn!(
+                                path = %p.display(),
+                                "Tokenizer path contains non-UTF-8 characters, skipping basetenkenizer; falling back to HuggingFace"
+                            );
+                            Arc::new(wrap_hf(hf))
                         }
-                    } else {
-                        tracing::warn!(
-                            path = %p.display(),
-                            "Tokenizer path contains non-UTF-8 characters, skipping fastokens; falling back to HuggingFace"
-                        );
-                        Arc::new(wrap_hf(hf))
                     }
-                } else {
-                    Arc::new(wrap_hf(hf))
                 };
 
                 if cache_enabled {