From b36673bb79345d1118dfb660536922492b0fb40b Mon Sep 17 00:00:00 2001 From: Harry Mellor <19981378+hmellor@users.noreply.github.com> Date: Thu, 24 Sep 2026 12:51:50 +0200 Subject: [PATCH] Remove the slow tokenizer mode Transformers v5 removed slow tokenizers and AutoTokenizer ignores `use_fast`, so `tokenizer_mode="slow"` behaved identically to "hf". Remove the mode and the remaining tokenizer `use_fast` arguments. Co-Authored-By: Claude Opus 5.5 (1M context) Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> --- benchmarks/backend_request_func.py | 4 ---- .../vision_language_multi_image_offline.py | 1 - rust/src/bench/src/cli.rs | 2 +- tests/renderers/test_token_offsets.py | 2 +- tests/tokenizers_/test_basic.py | 2 +- tests/tokenizers_/test_registry.py | 21 +++++-------------- vllm/benchmarks/serve.py | 1 - vllm/config/model.py | 2 -- vllm/entrypoints/llm.py | 4 ++-- vllm/renderers/hf.py | 3 +-- vllm/tokenizers/registry.py | 7 ------- .../processors/moondream3.py | 1 - 12 files changed, 11 insertions(+), 39 deletions(-) diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py index bae0149f106f..b8e5f5be4c13 100644 --- a/benchmarks/backend_request_func.py +++ b/benchmarks/backend_request_func.py @@ -630,10 +630,6 @@ def get_tokenizer( pretrained_model_name_or_path ): pretrained_model_name_or_path = get_model(pretrained_model_name_or_path) - if tokenizer_mode == "slow": - if kwargs.get("use_fast", False): - raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") - kwargs["use_fast"] = False if tokenizer_mode == "mistral": try: from vllm.tokenizers.mistral import MistralTokenizer diff --git a/examples/generate/multimodal/vision_language_multi_image_offline.py b/examples/generate/multimodal/vision_language_multi_image_offline.py index cccb75d838ce..7565f1e42b51 100644 --- a/examples/generate/multimodal/vision_language_multi_image_offline.py +++ b/examples/generate/multimodal/vision_language_multi_image_offline.py @@ -55,7 +55,6 @@ def load_aria(question: str, image_urls: list[str]) -> ModelRequestData: model_name = "rhymes-ai/Aria" engine_args = EngineArgs( model=model_name, - tokenizer_mode="slow", trust_remote_code=True, dtype="bfloat16", limit_mm_per_prompt={"image": len(image_urls)}, diff --git a/rust/src/bench/src/cli.rs b/rust/src/bench/src/cli.rs index fc0f5a29e483..302dbb479231 100644 --- a/rust/src/bench/src/cli.rs +++ b/rust/src/bench/src/cli.rs @@ -176,7 +176,7 @@ pub struct BenchServeArgs { #[arg(long)] pub tokenizer: Option, - /// Tokenizer mode (auto, hf, slow, mistral). Accepted for Python CLI + /// Tokenizer mode (auto, hf, mistral). Accepted for Python CLI /// compatibility; non-auto values are ignored with a warning. #[arg(long, default_value = "auto")] pub tokenizer_mode: String, diff --git a/tests/renderers/test_token_offsets.py b/tests/renderers/test_token_offsets.py index fc9192bb43e7..aad91655ba03 100644 --- a/tests/renderers/test_token_offsets.py +++ b/tests/renderers/test_token_offsets.py @@ -20,7 +20,7 @@ def fast_tokenizer(): """gpt2 ships a Fast tokenizer; use it to test the offsets happy path.""" from transformers import AutoTokenizer - return AutoTokenizer.from_pretrained("openai-community/gpt2", use_fast=True) + return AutoTokenizer.from_pretrained("openai-community/gpt2") def _make_base_renderer_with(tokenizer): diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py index 4da6381a0bc9..4a1a2b156050 100644 --- a/tests/tokenizers_/test_basic.py +++ b/tests/tokenizers_/test_basic.py @@ -23,7 +23,7 @@ def _assert_tokenizer_like(tokenizer: object): def test_tokenizer_like_protocol(): - tokenizer = get_tokenizer("openai-community/gpt2", use_fast=True) + tokenizer = get_tokenizer("openai-community/gpt2") assert isinstance(tokenizer, TokenizersBackend) _assert_tokenizer_like(tokenizer) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index b7201c1cfe4c..a4ee4a755e34 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -69,21 +69,11 @@ def test_resolve_tokenizer_args_idempotent(runner_type): ) -@pytest.mark.parametrize( - ("tokenizer_mode", "input_kwargs"), - [ - ("hf", {}), - ("hf", {"mistral_format": False}), - ("slow", {}), - ("slow", {"mistral_format": False}), - ], -) -def test_resolve_tokenizer_args_forces_hf_mistral_format_false( - tokenizer_mode, input_kwargs -): +@pytest.mark.parametrize("input_kwargs", [{}, {"mistral_format": False}]) +def test_resolve_tokenizer_args_forces_hf_mistral_format_false(input_kwargs): resolved_mode, _, _, kwargs = resolve_tokenizer_args( "mistralai/Mistral-Nemo-Instruct-2407", - tokenizer_mode=tokenizer_mode, + tokenizer_mode="hf", **input_kwargs, ) @@ -91,15 +81,14 @@ def test_resolve_tokenizer_args_forces_hf_mistral_format_false( assert kwargs["mistral_format"] is False -@pytest.mark.parametrize("tokenizer_mode", ["hf", "slow"]) -def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(tokenizer_mode): +def test_resolve_tokenizer_args_rejects_hf_mistral_format_true(): with pytest.raises( ValueError, match="mistral_format=True is not supported with tokenizer_mode='hf'", ): resolve_tokenizer_args( "mistralai/Mistral-Nemo-Instruct-2407", - tokenizer_mode=tokenizer_mode, + tokenizer_mode="hf", mistral_format=True, ) diff --git a/vllm/benchmarks/serve.py b/vllm/benchmarks/serve.py index 14c72386a5d0..d751f7f22dd7 100644 --- a/vllm/benchmarks/serve.py +++ b/vllm/benchmarks/serve.py @@ -1695,7 +1695,6 @@ def add_cli_args(parser: FlexibleArgumentParser): - "auto" will use the tokenizer from `mistral_common` for Mistral models if available, otherwise it will use the "hf" tokenizer.\n - "hf" will use the fast tokenizer if available.\n - - "slow" will always use the slow tokenizer.\n - "mistral" will always use the tokenizer from `mistral_common`.\n - "deepseek_v32" will always use the tokenizer from `deepseek_v32`.\n - Other custom values can be supported via plugins.""", diff --git a/vllm/config/model.py b/vllm/config/model.py index 7b938bc8e55d..43876182b79b 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -89,7 +89,6 @@ TokenizerMode = Literal[ "auto", "hf", - "slow", "mistral", "deepseek_v32", "deepseek_v4", @@ -148,7 +147,6 @@ class ModelConfig: - "auto" will use the tokenizer from `mistral_common` for Mistral models if available, otherwise it will use the "hf" tokenizer. - "hf" will use the fast tokenizer if available. - - "slow" will always use the slow tokenizer. - "mistral" will always use the tokenizer from `mistral_common`. - "deepseek_v32" will always use the tokenizer from `deepseek_v32`. - "deepseek_v4" will always use the tokenizer from `deepseek_v4`. diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 999310aa6169..c0720ad8baee 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -76,8 +76,8 @@ class LLM(BeamSearchOfflineMixin, PoolingOfflineMixin, OfflineInferenceMixin): Args: model: The name or path of a HuggingFace Transformers model. tokenizer: The name or path of a HuggingFace Transformers tokenizer. - tokenizer_mode: The tokenizer mode. "auto" will use the fast tokenizer - if available, and "slow" will always use the slow tokenizer. + tokenizer_mode: The tokenizer mode. See + [ModelConfig.tokenizer_mode][vllm.config.ModelConfig.tokenizer_mode]. skip_tokenizer_init: If true, skip initialization of tokenizer and detokenizer. Expect valid prompt_token_ids and None for prompt from the input. diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 36605c62b914..4ef14aee58cf 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -966,8 +966,7 @@ def __init__( ) def _can_produce_offsets(self) -> bool: - # HF tokenizers may be slow (use_fast=False); only fast tokenizers - # expose offset_mapping. + # Only fast tokenizers expose offset_mapping. return self.tokenizer is not None and self.tokenizer.is_fast def render_messages( diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 46446ecd6df7..09a715c117de 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -145,13 +145,6 @@ def resolve_tokenizer_args( else: assert_never(runner_type) - if tokenizer_mode == "slow": - if kwargs.get("use_fast", False): - raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") - - tokenizer_mode = "hf" - kwargs["use_fast"] = False - # Try to use official Mistral tokenizer if possible if ( tokenizer_mode == "auto" diff --git a/vllm/transformers_utils/processors/moondream3.py b/vllm/transformers_utils/processors/moondream3.py index 22bdeef21abe..321fc9517172 100644 --- a/vllm/transformers_utils/processors/moondream3.py +++ b/vllm/transformers_utils/processors/moondream3.py @@ -209,7 +209,6 @@ def from_pretrained( "revision", "subfolder", "token", - "use_fast", ): if key in kwargs: tokenizer_kwargs[key] = kwargs[key]