Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 116 additions & 2 deletions tests/tokenizers_/test_basic.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
from types import SimpleNamespace
from typing import _get_protocol_attrs # type: ignore
from unittest.mock import patch

Expand All @@ -15,7 +16,11 @@
from vllm.tokenizers import TokenizerLike, get_tokenizer
from vllm.tokenizers.hf import HfTokenizer
from vllm.tokenizers.mistral import MistralTokenizer
from vllm.tokenizers.registry import resolve_tokenizer_args
from vllm.tokenizers.registry import (
cached_tokenizer_from_config,
resolve_tokenizer_args,
tokenizer_args_from_config,
)


def _get_missing_attrs(obj: object, target: type):
Expand Down Expand Up @@ -65,7 +70,11 @@ def test_tokenizer_revision(tokenizer_name: str):
@pytest.mark.parametrize(
"code_revision, error_match",
[
pytest.param("never", "not a valid git identifier", id="unknown-revision"),
pytest.param(
"never",
"not a valid git identifier|couldn't find them in the cached files",
id="unknown-revision",
),
pytest.param(
"73293ac504e43f4a895688b57c65cebe24f01a38",
r"tokenization_fast\.py",
Expand Down Expand Up @@ -143,6 +152,111 @@ def test_tokenizer_code_revision_default_or_matching(code_revision: str | None):
_assert_tokenizer_like(tokenizer)


@pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
)
@pytest.mark.parametrize(
"model, code_revision, kwargs, error_match",
[
pytest.param(
None,
"never",
{},
"not a valid git identifier|couldn't find them in the cached files",
id="same-repo",
),
pytest.param(
None,
"73293ac504e43f4a895688b57c65cebe24f01a38",
{},
r"tokenization_fast\.py",
id="same-repo-missing-code",
),
pytest.param(
"facebook/opt-125m",
"27dcfa74d334bc871f3234de431e71c6eeba5dd6",
{},
None,
id="separate-repo",
),
pytest.param(
None,
"never",
{"code_revision": "08a35780b7a78794436d8654aafa26ee0165c149"},
None,
id="explicit-revision",
),
pytest.param(None, "never", {"code_revision": None}, None, id="explicit-none"),
],
)
def test_config_tokenizer_code_revision(
loader, model, code_revision, kwargs, error_match
):
"""Only same-repo tokenizers inherit model code revisions; overrides win."""
revision = "08a35780b7a78794436d8654aafa26ee0165c149"
repo = "hf-internal-testing/test_dynamic_tokenizer"
config = SimpleNamespace(
model=model or repo,
tokenizer=repo,
runner_type="generate",
tokenizer_mode="hf",
tokenizer_revision=revision,
code_revision=code_revision,
trust_remote_code=True,
skip_tokenizer_init=False,
)

def load():
result = loader(config, **kwargs)
if loader is tokenizer_args_from_config:
mode, name, args, resolved_kwargs = result
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
return result

if error_match is not None:
with pytest.raises(OSError, match=error_match):
load()
else:
tokenizer = load()
reference = AutoTokenizer.from_pretrained(
repo,
revision=revision,
code_revision=kwargs.get("code_revision"),
trust_remote_code=True,
)
assert tokenizer.encode("hello world") == reference.encode("hello world")
_assert_tokenizer_like(tokenizer)


@pytest.mark.parametrize(
"loader", (cached_tokenizer_from_config, tokenizer_args_from_config)
)
@pytest.mark.parametrize("tokenizer_mode", ["mistral", "auto"])
@pytest.mark.parametrize("code_revision", [None, "never"])
def test_config_mistral_code_revision(loader, tokenizer_mode, code_revision):
"""Config loading must not pass HF-only revisions to the Mistral backend."""
repo = "mistralai/Mistral-7B-Instruct-v0.3"
config = SimpleNamespace(
model=repo,
tokenizer=repo,
runner_type="generate",
tokenizer_mode=tokenizer_mode,
tokenizer_revision=None,
code_revision=code_revision,
trust_remote_code=False,
skip_tokenizer_init=False,
)
tokenizer = loader(config)
if loader is tokenizer_args_from_config:
mode, name, args, kwargs = tokenizer
tokenizer = get_tokenizer(name, *args, tokenizer_mode=mode, **kwargs)

reference = get_tokenizer(repo, tokenizer_mode="mistral")
assert isinstance(tokenizer, MistralTokenizer)
assert tokenizer.encode("hello world") == reference.encode("hello world")


def test_hub_tokenizer_loads_from_local_snapshot():
"""Transformers makes a Hub request per file when given a repo ID."""
with patch.object(
Expand Down
119 changes: 118 additions & 1 deletion tests/tokenizers_/test_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,15 @@
from transformers import AutoConfig, Qwen3_5MoeConfig
from transformers.models.auto.configuration_auto import CONFIG_MAPPING

from vllm.tokenizers import TokenizerLike
from vllm.tokenizers import TokenizerLike, registry
from vllm.tokenizers.registry import (
TokenizerRegistry,
cached_get_tokenizer,
cached_resolve_tokenizer_args,
cached_tokenizer_from_config,
get_tokenizer,
resolve_tokenizer_args,
tokenizer_args_from_config,
)


Expand Down Expand Up @@ -116,11 +117,13 @@ def test_cached_tokenizer_from_config_registers_local_config(tmp_path: Path):
)

model_config = SimpleNamespace(
model=str(tmp_path),
skip_tokenizer_init=False,
tokenizer=str(tmp_path),
runner_type="generate",
tokenizer_mode="hf",
tokenizer_revision=None,
code_revision=None,
trust_remote_code=True,
hf_config=Qwen3_5MoeConfig(),
)
Expand Down Expand Up @@ -164,3 +167,117 @@ def fake_from_pretrained(path_or_repo_id: str, *args, **kwargs):
CONFIG_MAPPING._extra_content.pop("qwen3_5_moe", None)
if registered_config is not None:
CONFIG_MAPPING._extra_content["qwen3_5_moe"] = registered_config


@pytest.fixture
def config_tokenizer_backend(monkeypatch):
"""Keep config helpers and dispatch real; replace Hub/backend loading."""
calls = []

class Backend:
@classmethod
def from_pretrained(cls, name, *args, **kwargs):
calls.append(kwargs)
return SimpleNamespace(is_fast=True)

monkeypatch.setattr(registry, "get_config", lambda *a, **kw: None)
monkeypatch.setattr(registry, "_maybe_register_hf_config", lambda config: None)
monkeypatch.setattr(registry, "is_mistral_model_repo", lambda **kw: True)
monkeypatch.setattr(registry, "any_pattern_in_repo_files", lambda **kw: True)
monkeypatch.setattr(
registry, "get_non_weight_snapshot_path", lambda name, **kw: name
)
monkeypatch.setattr(TokenizerRegistry, "load_tokenizer_cls", lambda mode: Backend)
cached_get_tokenizer.cache_clear()
cached_resolve_tokenizer_args.cache_clear()
yield calls
cached_get_tokenizer.cache_clear()
cached_resolve_tokenizer_args.cache_clear()


def _load_config_tokenizer(loader, config, **kwargs):
result = loader(config, **kwargs)
if loader is tokenizer_args_from_config:
mode, name, args, resolved_kwargs = result
return get_tokenizer(name, *args, tokenizer_mode=mode, **resolved_kwargs)
return result


@pytest.mark.parametrize(
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
)
@pytest.mark.parametrize("mode", ["mistral", "auto"])
@pytest.mark.parametrize("code_revision", [None, "model-code"])
@pytest.mark.parametrize("overrides", [{}, {"code_revision": None}])
def test_config_mistral_does_not_forward_code_revision(
loader, mode, code_revision, overrides, config_tokenizer_backend
):
"""HF code revisions must not reach MistralCommonBackend's strict kwargs."""
config = SimpleNamespace(
model="test-model",
tokenizer="test-model",
runner_type="generate",
tokenizer_mode=mode,
tokenizer_revision=None,
code_revision=code_revision,
trust_remote_code=False,
skip_tokenizer_init=False,
)
_load_config_tokenizer(loader, config, **overrides)
assert len(config_tokenizer_backend) == 1
assert "code_revision" not in config_tokenizer_backend[0]


@pytest.mark.parametrize(
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
)
@pytest.mark.parametrize("missing", [False, True])
def test_config_tokenizer_without_code_revision(
loader, missing, config_tokenizer_backend
):
"""Default or minimal configs must not introduce an HF-only plugin kwarg."""
config = SimpleNamespace(
model="test-model",
tokenizer="test-model",
runner_type="generate",
tokenizer_mode="test_tokenizer",
tokenizer_revision=None,
trust_remote_code=False,
skip_tokenizer_init=False,
)
if not missing:
config.code_revision = None
_load_config_tokenizer(loader, config)
assert len(config_tokenizer_backend) == 1
assert "code_revision" not in config_tokenizer_backend[0]


@pytest.mark.parametrize(
"loader", (tokenizer_args_from_config, cached_tokenizer_from_config)
)
@pytest.mark.parametrize("mode", ["hf", "cohere", "kimi_k3", "inkling", "deepseek_v4"])
@pytest.mark.parametrize(
"overrides, expected",
[
({}, "model-code"),
({"code_revision": None}, None),
({"code_revision": "override"}, "override"),
],
)
def test_config_hf_backed_tokenizer_preserves_code_revision(
loader, mode, overrides, expected, config_tokenizer_backend
):
"""Filtering Mistral kwargs must preserve HF wrappers and caller overrides."""
config = SimpleNamespace(
model="test-model",
tokenizer="test-model",
runner_type="generate",
tokenizer_mode=mode,
tokenizer_revision=None,
code_revision="model-code",
trust_remote_code=True,
skip_tokenizer_init=False,
)
_load_config_tokenizer(loader, config, **overrides)
assert len(config_tokenizer_backend) == 1
assert config_tokenizer_backend[0]["code_revision"] == expected
12 changes: 12 additions & 0 deletions vllm/tokenizers/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -173,13 +173,21 @@ def resolve_tokenizer_args(
token=kwargs.get("token"),
)

if tokenizer_mode == "mistral":
# MistralCommonBackend rejects HF-only keyword arguments, even None.
kwargs.pop("code_revision", None)

return tokenizer_mode, tokenizer_name, args, kwargs


cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args)


def tokenizer_args_from_config(config: "ModelConfig", **kwargs):
if config.tokenizer == config.model:
code_revision = getattr(config, "code_revision", None)
if code_revision is not None:
kwargs.setdefault("code_revision", code_revision)
return cached_resolve_tokenizer_args(
config.tokenizer,
runner_type=config.runner_type,
Expand Down Expand Up @@ -287,6 +295,10 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs):
return None

_maybe_register_hf_config(getattr(model_config, "hf_config", None))
if model_config.tokenizer == model_config.model:
code_revision = getattr(model_config, "code_revision", None)
if code_revision is not None:
kwargs.setdefault("code_revision", code_revision)

return cached_get_tokenizer(
model_config.tokenizer,
Expand Down
Loading