[Bugfix] Honor code_revision in config-based tokenizer loading

tests/tokenizers_/test_basic.py

@@ -1,6 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 from pathlib import Path
+from types import SimpleNamespace
 from typing import _get_protocol_attrs  # type: ignore
 from unittest.mock import patch
 
@@ -15,7 +16,11 @@
 from vllm.tokenizers import TokenizerLike, get_tokenizer
 from vllm.tokenizers.hf import HfTokenizer
 from vllm.tokenizers.mistral import MistralTokenizer
-from vllm.tokenizers.registry import resolve_tokenizer_args
+from vllm.tokenizers.registry import (
+    cached_tokenizer_from_config,
+    resolve_tokenizer_args,
+    tokenizer_args_from_config,
+)
 
 
 def _get_missing_attrs(obj: object, target: type):
@@ -65,7 +70,11 @@ def test_tokenizer_revision(tokenizer_name: str):
 @pytest.mark.parametrize(
     "code_revision, error_match",
     [
-        pytest.param("never", "not a valid git identifier", id="unknown-revision"),
+        pytest.param(
+            "never",
+            "not a valid git identifier|couldn't find them in the cached files",
+            id="unknown-revision",
+        ),
         pytest.param(
             "73293ac504e43f4a895688b57c65cebe24f01a38",
             r"tokenization_fast\.py",
@@ -143,6 +152,83 @@ def test_tokenizer_code_revision_default_or_matching(code_revision: str | None):
     _assert_tokenizer_like(tokenizer)
 
 
+@pytest.mark.skip_global_cleanup
+@pytest.mark.parametrize(
+    "loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
+)
+@pytest.mark.parametrize(
+    "model, code_revision, kwargs, error_match",
+    [
+        pytest.param(
+            None,
+            "never",
+            {},
+            "not a valid git identifier|couldn't find them in the cached files",
+            id="same-repo",
+        ),
+        pytest.param(
+            None,
+            "73293ac504e43f4a895688b57c65cebe24f01a38",
+            {},
+            r"tokenization_fast\.py",
+            id="same-repo-missing-code",
+        ),
+        pytest.param(
+            "facebook/opt-125m",
+            "27dcfa74d334bc871f3234de431e71c6eeba5dd6",
+            {},
+            None,
+            id="separate-repo",
+        ),
+        pytest.param(
+            None,
+            "never",
+            {"code_revision": "08a35780b7a78794436d8654aafa26ee0165c149"},
+            None,
+            id="explicit-revision",
+        ),
+        pytest.param(None, "never", {"code_revision": None}, None, id="explicit-none"),
+    ],
+)
+def test_config_tokenizer_code_revision(
+    loader, model, code_revision, kwargs, error_match
+):
+    """Only same-repo tokenizers inherit model code revisions; overrides win."""
+    revision = "08a35780b7a78794436d8654aafa26ee0165c149"
+    repo = "hf-internal-testing/test_dynamic_tokenizer"
+    config = SimpleNamespace(
+        model=model or repo,
+        tokenizer=repo,
+        runner_type="generate",
+        tokenizer_mode="hf",
+        tokenizer_revision=revision,
+        code_revision=code_revision,
+        trust_remote_code=True,
+        skip_tokenizer_init=False,
+    )
+
+    def load():
+        result = loader(config, **kwargs)
+        if loader is tokenizer_args_from_config:
+            mode, name, args, resolved_kwargs = result
+            return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
+        return result
+
+    if error_match is not None:
+        with pytest.raises(OSError, match=error_match):
+            load()
+    else:
+        tokenizer = load()
+        reference = AutoTokenizer.from_pretrained(
+            repo,
+            revision=revision,
+            code_revision=kwargs.get("code_revision"),
+            trust_remote_code=True,
+        )
+        assert tokenizer.encode("hello world") == reference.encode("hello world")
+        _assert_tokenizer_like(tokenizer)
+
+
 def test_hub_tokenizer_loads_from_local_snapshot():
     """Transformers makes a Hub request per file when given a repo ID."""
     with patch.object(

tests/tokenizers_/test_registry.py

@@ -116,11 +116,13 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
     )
 
     model_config = SimpleNamespace(
+        model=str(tmp_path),
         skip_tokenizer_init=False,
         tokenizer=str(tmp_path),
         runner_type="generate",
         tokenizer_mode="hf",
         tokenizer_revision=None,
+        code_revision=None,
         trust_remote_code=True,
         hf_config=Qwen3_5MoeConfig(),
     )

vllm/tokenizers/registry.py

@@ -180,6 +180,10 @@ def resolve_tokenizer_args(
 
 
 def tokenizer_args_from_config(config: "ModelConfig", **kwargs):
+    kwargs.setdefault(
+        "code_revision",
+        config.code_revision if config.tokenizer == config.model else None,
+    )
     return cached_resolve_tokenizer_args(
         config.tokenizer,
         runner_type=config.runner_type,
@@ -287,6 +291,12 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
         return None
 
     _maybe_register_hf_config(getattr(model_config, "hf_config", None))
+    kwargs.setdefault(
+        "code_revision",
+        model_config.code_revision
+        if model_config.tokenizer == model_config.model
+        else None,
+    )
 
     return cached_get_tokenizer(
         model_config.tokenizer,