feat(frontend): make tokenizer fallback configurable

components/src/dynamo/common/configuration/utils.py

@@ -11,6 +11,16 @@
 T = TypeVar("T")
 
 
+def parse_bool(value: str) -> bool:
+    """Parse Dynamo's truthy and falsy configuration values."""
+    normalized = value.strip().lower()
+    if normalized in ("1", "true", "on", "yes"):
+        return True
+    if normalized in ("", "0", "false", "off", "no"):
+        return False
+    raise argparse.ArgumentTypeError("expected one of: true/false, 1/0, on/off, yes/no")
+
+
 def split_served_model_names(served_model_name: Any) -> list[str]:
     """Split a ``--served-model-name`` value into individual names.
 
@@ -98,6 +108,7 @@ def add_argument(
     help: str,
     obsolete_flag: Optional[str] = None,
     arg_type: Optional[Union[type, Callable[..., Any]]] = str,
+    env_value_type: Optional[Union[type, Callable[..., Any]]] = None,
     **kwargs: Any,
 ) -> None:
     """
@@ -114,10 +125,11 @@ def add_argument(
         dest: Optional destination name (defaults to flag_name with dashes replaced by underscores)
         choices: Optional list of valid values for the argument.
         arg_type: Type for the argument (default: str)
+        env_value_type: Optional parser used only for the environment value
     """
     arg_dest = _get_dest_name(flag_name, kwargs.get("dest"))
-    value_type_for_env: Optional[Union[type, Callable[..., Any]]] = None
-    if arg_type is not None and callable(arg_type):
+    value_type_for_env = env_value_type
+    if value_type_for_env is None and arg_type is not None and callable(arg_type):
         value_type_for_env = arg_type
     if isinstance(default, list) and (arg_type is None or arg_type is str):
         value_type_for_env = None
@@ -152,6 +164,7 @@ def add_negatable_bool_argument(
     help: str,
     dest: Optional[str] = None,
     obsolete_flag: Optional[str] = None,
+    env_value_type: Optional[Callable[..., bool]] = None,
 ) -> None:
     """
     Add negatable boolean flag (--foo / --no-foo).
@@ -164,6 +177,7 @@ def add_negatable_bool_argument(
         help: Help text
         dest: Optional destination name for the parsed value
         obsolete_flag: Optional obsolete/legacy flag (for help msg only, must start with '--')
+        env_value_type: Optional strict parser for the environment value
     """
     add_argument(
         parser,
@@ -174,6 +188,7 @@ def add_negatable_bool_argument(
         dest=dest,
         obsolete_flag=obsolete_flag,
         arg_type=None,
+        env_value_type=env_value_type,
         action=argparse.BooleanOptionalAction,
     )
 

components/src/dynamo/common/tests/configuration/test_utils.py

@@ -11,6 +11,7 @@
     add_negatable_bool_argument,
     env_or_default,
     nullable_float,
+    parse_bool,
 )
 
 pytestmark = [
@@ -20,6 +21,28 @@
 ]
 
 
+@pytest.mark.parametrize(
+    ("value", "expected"),
+    [
+        ("true", True),
+        ("1", True),
+        ("on", True),
+        ("yes", True),
+        ("false", False),
+        ("0", False),
+        ("off", False),
+        ("no", False),
+    ],
+)
+def test_parse_bool(value, expected):
+    assert parse_bool(value) is expected
+
+
+def test_parse_bool_rejects_invalid_value():
+    with pytest.raises(argparse.ArgumentTypeError, match="expected one of"):
+        parse_bool("flase")
+
+
 class TestEnvOrDefault:
     """Test env_or_default function."""
 
@@ -251,6 +274,20 @@ def test_uses_env_var_when_set(self, monkeypatch):
         args = parser.parse_args([])
         assert args.enable_feature is False
 
+    def test_strict_env_parser_rejects_invalid_value(self, monkeypatch):
+        monkeypatch.setenv("TEST_ENABLE", "flase")
+        parser = argparse.ArgumentParser()
+
+        with pytest.raises(argparse.ArgumentTypeError, match="expected one of"):
+            add_negatable_bool_argument(
+                parser,
+                flag_name="--enable-feature",
+                env_var="TEST_ENABLE",
+                default=True,
+                help="Enable feature",
+                env_value_type=parse_bool,
+            )
+
     def test_converts_hyphens_to_underscores(self):
         """Test that flag name with hyphens converts to underscores in dest."""
         parser = argparse.ArgumentParser()

components/src/dynamo/frontend/frontend_args.py

@@ -24,6 +24,7 @@
     add_argument,
     add_negatable_bool_argument,
     env_or_default,
+    parse_bool,
 )
 
 from . import __version__
@@ -89,6 +90,7 @@ class FrontendConfig(RouterConfigBase, KvRouterConfigBase, AicPerfConfigBase):
     exclude_tools_when_tool_choice_none: bool
     preprocess_workers: int
     tokenizer_backend: str
+    tokenizer_fallback: bool
     trust_remote_code: bool
     frontend_route_extensions: list[str]
 
@@ -552,6 +554,23 @@ def add_arguments(self, parser) -> None:
             choices=["default", "fastokens", "basetenkenizer"],
         )
 
+        add_negatable_bool_argument(
+            g,
+            flag_name="--tokenizer-fallback",
+            env_var="DYN_TOKENIZER_FALLBACK",
+            default=True,
+            help=(
+                "Automatic fallback to HuggingFace is deprecated and will be "
+                "disabled by default in a future release. The current behavior "
+                "falls back when the selected fastokens or basetenkenizer backend "
+                "cannot load the model tokenizer. Use "
+                "--no-tokenizer-fallback to fail model initialization instead. "
+                "In dynamic mode, discovery retries the load while the frontend "
+                "continues running."
+            ),
+            env_value_type=parse_bool,
+        )
+
         add_negatable_bool_argument(
             g,
             flag_name="--trust-remote-code",

components/src/dynamo/frontend/main.py

@@ -412,6 +412,7 @@ def signal_handler():
         "enable_streaming_reasoning_dispatch": config.enable_streaming_reasoning_dispatch,
         "reasoning_field_name": config.reasoning_field_name,
         "tokenizer_backend": config.tokenizer_backend,
+        "tokenizer_fallback": config.tokenizer_fallback,
     }
     if config.migration_max_seq_len is not None:
         kwargs["migration_max_seq_len"] = config.migration_max_seq_len

docs/fern/pages/developer-guide/knowledge-base/modular-components/frontend/tokenizer.md

@@ -31,17 +31,22 @@ Use this backend for supported `tokenizer.json` models when you need Baseten Tok
 #### Compatibility notes:
 
 - Works with standard BPE `tokenizer.json` files (Qwen, LLaMA, GPT-family, Mistral, DeepSeek, etc.).
-- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace; requests are never dropped.
+- If `fastokens` or `basetenkenizer` cannot load a particular tokenizer file, the frontend logs a warning and transparently falls back to HuggingFace by default. Use `--no-tokenizer-fallback` to reject incompatible tokenizers during model initialization.
 - Special tokens declared only in a sibling `tokenizer_config.json` are preserved for Baseten encoding and decoding and for Dynamo's L1 prefix-cache boundaries.
 - Has no effect on TikToken-format tokenizers (`.model` / `.tiktoken` files), which always use the TikToken backend.
 
 ## Configuration
 
 Set the backend with a CLI flag or environment variable. The CLI flag takes precedence.
 
+> [!WARNING]
+> Automatic tokenizer fallback is deprecated and will be disabled by default in a future release.
+> Set `--no-tokenizer-fallback` or `DYN_TOKENIZER_FALLBACK=false` to adopt the future behavior now.
+
 | CLI Argument | Env Var | Valid values | Default |
 |---|---|---|---|
 | `--tokenizer` | `DYN_TOKENIZER` | `default`, `fastokens`, `basetenkenizer` | `default` |
+| `--tokenizer-fallback` / `--no-tokenizer-fallback` | `DYN_TOKENIZER_FALLBACK` | `true`/`false`, `1`/`0`, `on`/`off`, `yes`/`no` | `true` |
 
 **Examples:**
 
@@ -55,6 +60,9 @@ python -m dynamo.frontend
 
 # Baseten Tokenizer
 python -m dynamo.frontend --tokenizer basetenkenizer
+
+# Require Baseten Tokenizer instead of falling back to HuggingFace
+python -m dynamo.frontend --tokenizer basetenkenizer --no-tokenizer-fallback
 ```
 
 ## Dynamo Frontend Behavior
@@ -64,5 +72,5 @@ When a non-default backend is selected:
 1. The frontend resolves `--tokenizer` / `DYN_TOKENIZER` and passes the selected backend to the Rust runtime.
 2. `ModelDeploymentCard::tokenizer()` loads the HuggingFace tokenizer first for fallback behavior and L1 cache special-token metadata.
 3. Dynamo constructs `FastTokenizer` for `fastokens` or `BasetenTokenizer` for `basetenkenizer` from the same `tokenizer.json` file.
-4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace.
+4. If construction fails because the tokenizer uses unsupported features, Dynamo logs a warning and falls back to HuggingFace. With `--no-tokenizer-fallback`, model initialization fails and reports the backend loading error instead. In dynamic mode, discovery retries the load while the frontend continues running.
 5. When the L1 prefix cache is enabled, Dynamo wraps the selected backend with the same special-token boundary metadata and cache metrics used by the default path.