From bd9e0b37bdbb31b776a4c2e8993576873878d108 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 17:17:56 +0000 Subject: [PATCH 01/31] [Misc] Convert `TokenizerBase` to protocol, consolidate tokenizer tests Signed-off-by: DarkLight1337 --- .buildkite/test-amd.yaml | 3 - .buildkite/test-pipeline.yaml | 3 - docs/features/reasoning_outputs.md | 5 +- docs/features/tool_calling.md | 2 +- .../openai/tool_parsers/conftest.py | 4 +- .../tool_parsers/test_hermes_tool_parser.py | 12 +- .../test_llama3_json_tool_parser.py | 4 +- .../test_llama4_pythonic_tool_parser.py | 10 +- .../tool_parsers/test_olmo3_tool_parser.py | 10 +- .../tool_parsers/test_pythonic_tool_parser.py | 10 +- .../entrypoints/openai/tool_parsers/utils.py | 4 +- .../multimodal/generation/vlm_utils/core.py | 4 +- .../multimodal/generation/vlm_utils/types.py | 4 +- tests/multimodal/test_processing.py | 16 +- tests/tokenization/__init__.py | 0 tests/tokenization/test_do_lower_case.py | 18 - tests/tokenization/test_tokenizer.py | 23 - tests/tokenization/test_tokenizer_registry.py | 124 ---- .../tool_use/test_ernie45_moe_tool_parser.py | 4 +- tests/tool_use/test_jamba_tool_parser.py | 6 +- tests/tool_use/test_qwen3coder_tool_parser.py | 4 +- tests/tool_use/test_seed_oss_tool_parser.py | 4 +- tests/tool_use/test_xlam_tool_parser.py | 4 +- .../test_cached_tokenizer.py | 5 +- .../test_detokenize.py | 0 .../test_get_eos.py | 0 .../test_mistral_tokenizer.py | 20 - ...gs_from_processor.py => test_processor.py} | 0 .../{test_config.py => test_repo_utils.py} | 0 tests/transformers_utils/test_tokenizer.py | 53 ++ .../test_tokenizer_registry.py | 36 + tests/v1/engine/test_output_processor.py | 4 +- tools/pre_commit/check_pickle_imports.py | 2 +- vllm/benchmarks/datasets.py | 4 +- vllm/engine/protocol.py | 4 +- vllm/entrypoints/chat_utils.py | 12 +- vllm/entrypoints/llm.py | 10 +- vllm/entrypoints/openai/serving_chat.py | 10 +- vllm/entrypoints/openai/serving_completion.py | 17 +- vllm/entrypoints/openai/serving_engine.py | 36 +- vllm/entrypoints/openai/serving_responses.py | 20 +- vllm/entrypoints/openai/serving_score.py | 8 +- .../openai/serving_tokenization.py | 4 +- .../tool_parsers/abstract_tool_parser.py | 4 +- .../tool_parsers/deepseekv31_tool_parser.py | 4 +- .../tool_parsers/deepseekv3_tool_parser.py | 4 +- .../tool_parsers/ernie45_tool_parser.py | 4 +- .../tool_parsers/glm4_moe_tool_parser.py | 4 +- .../granite_20b_fc_tool_parser.py | 4 +- .../tool_parsers/granite_tool_parser.py | 4 +- .../openai/tool_parsers/hermes_tool_parser.py | 4 +- .../tool_parsers/hunyuan_a13b_tool_parser.py | 4 +- .../tool_parsers/internlm2_tool_parser.py | 4 +- .../openai/tool_parsers/jamba_tool_parser.py | 4 +- .../tool_parsers/kimi_k2_tool_parser.py | 4 +- .../tool_parsers/longcat_tool_parser.py | 4 +- .../tool_parsers/minimax_m2_tool_parser.py | 4 +- .../tool_parsers/minimax_tool_parser.py | 4 +- .../tool_parsers/mistral_tool_parser.py | 6 +- .../openai/tool_parsers/openai_tool_parser.py | 6 +- .../tool_parsers/qwen3coder_tool_parser.py | 4 +- .../tool_parsers/qwen3xml_tool_parser.py | 4 +- .../tool_parsers/seed_oss_tool_parser.py | 4 +- .../openai/tool_parsers/step3_tool_parser.py | 4 +- .../openai/tool_parsers/xlam_tool_parser.py | 4 +- vllm/entrypoints/renderer.py | 10 +- vllm/entrypoints/score_utils.py | 6 +- vllm/inputs/preprocess.py | 10 +- vllm/logits_process.py | 4 +- vllm/model_executor/models/h2ovl.py | 4 +- vllm/model_executor/models/internvl.py | 6 +- .../model_executor/models/nano_nemotron_vl.py | 10 +- vllm/model_executor/models/nemotron_vl.py | 4 +- vllm/model_executor/models/opencua.py | 4 +- vllm/model_executor/models/qwen2_vl.py | 4 +- vllm/model_executor/models/skyworkr1v.py | 4 +- vllm/model_executor/models/step3_vl.py | 4 +- vllm/multimodal/processing.py | 46 +- vllm/multimodal/registry.py | 18 +- vllm/reasoning/abs_reasoning_parsers.py | 6 +- vllm/reasoning/basic_parsers.py | 4 +- vllm/reasoning/minimax_m2_reasoning_parser.py | 4 +- vllm/reasoning/olmo3_reasoning_parser.py | 4 +- vllm/sampling_params.py | 4 +- vllm/transformers_utils/detokenizer_utils.py | 10 +- vllm/transformers_utils/tokenizer.py | 43 +- vllm/transformers_utils/tokenizer_base.py | 93 +-- vllm/transformers_utils/tokenizers/mistral.py | 32 +- vllm/v1/engine/async_llm.py | 13 +- vllm/v1/engine/detokenizer.py | 6 +- vllm/v1/engine/input_processor.py | 637 ++++++++++++++++++ vllm/v1/engine/llm_engine.py | 13 +- vllm/v1/engine/logprobs.py | 6 +- vllm/v1/engine/output_processor.py | 9 +- vllm/v1/engine/processor.py | 8 +- vllm/v1/structured_output/backend_types.py | 6 +- vllm/v1/structured_output/utils.py | 10 +- 97 files changed, 1093 insertions(+), 564 deletions(-) delete mode 100644 tests/tokenization/__init__.py delete mode 100644 tests/tokenization/test_do_lower_case.py delete mode 100644 tests/tokenization/test_tokenizer.py delete mode 100644 tests/tokenization/test_tokenizer_registry.py rename tests/{tokenization => transformers_utils}/test_cached_tokenizer.py (85%) rename tests/{tokenization => transformers_utils}/test_detokenize.py (100%) rename tests/{tokenization => transformers_utils}/test_get_eos.py (100%) rename tests/{tokenization => transformers_utils}/test_mistral_tokenizer.py (98%) rename tests/transformers_utils/{test_get_processor_kwargs_from_processor.py => test_processor.py} (100%) rename tests/transformers_utils/{test_config.py => test_repo_utils.py} (100%) create mode 100644 tests/transformers_utils/test_tokenizer.py create mode 100644 tests/transformers_utils/test_tokenizer_registry.py create mode 100644 vllm/v1/engine/input_processor.py diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 4ddf11c0b268..b5758c9c4247 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -316,15 +316,12 @@ steps: source_file_dependencies: - vllm/ - tests/engine - - tests/tokenization - tests/test_sequence - tests/test_config - tests/test_logger - tests/test_vllm_port commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py - # OOM in the CI unless we run this separately - - pytest -v -s tokenization - label: V1 Test e2e + engine # 30min timeout_in_minutes: 45 diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 375645fde747..08a4c91fd91e 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -282,15 +282,12 @@ steps: source_file_dependencies: - vllm/ - tests/engine - - tests/tokenization - tests/test_sequence - tests/test_config - tests/test_logger - tests/test_vllm_port commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py - # OOM in the CI unless we run this separately - - pytest -v -s tokenization - label: V1 Test e2e + engine # 30min timeout_in_minutes: 45 diff --git a/docs/features/reasoning_outputs.md b/docs/features/reasoning_outputs.md index 5f26c7cf182b..08a0dd69efa9 100644 --- a/docs/features/reasoning_outputs.md +++ b/docs/features/reasoning_outputs.md @@ -216,14 +216,13 @@ You can add a new `ReasoningParser` similar to [vllm/reasoning/deepseek_r1_reaso # import the required packages from vllm.reasoning import ReasoningParser, ReasoningParserManager - from vllm.entrypoints.openai.protocol import (ChatCompletionRequest, - DeltaMessage) + from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage # define a reasoning parser and register it to vllm # the name list in register_module can be used # in --reasoning-parser. class ExampleParser(ReasoningParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) def extract_reasoning_streaming( diff --git a/docs/features/tool_calling.md b/docs/features/tool_calling.md index 22dda37279ac..b6dfbf10b456 100644 --- a/docs/features/tool_calling.md +++ b/docs/features/tool_calling.md @@ -422,7 +422,7 @@ Here is a summary of a plugin file: # in --tool-call-parser. you can define as many # tool parsers as you want here. class ExampleToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) # adjust request. e.g.: set skip special tokens diff --git a/tests/entrypoints/openai/tool_parsers/conftest.py b/tests/entrypoints/openai/tool_parsers/conftest.py index f2ac5e5b9a8f..f22e71393d99 100644 --- a/tests/entrypoints/openai/tool_parsers/conftest.py +++ b/tests/entrypoints/openai/tool_parsers/conftest.py @@ -4,9 +4,9 @@ import pytest from transformers import AutoTokenizer -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike @pytest.fixture(scope="function") -def default_tokenizer() -> AnyTokenizer: +def default_tokenizer() -> TokenizerLike: return AutoTokenizer.from_pretrained("gpt2") diff --git a/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py index 38008dafe32b..65d05e3b8899 100644 --- a/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py @@ -7,7 +7,7 @@ from vllm.entrypoints.openai.protocol import ChatCompletionRequest from vllm.entrypoints.openai.tool_parsers.hermes_tool_parser import Hermes2ProToolParser -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from ....utils import RemoteOpenAIServer @@ -270,14 +270,14 @@ async def test_streaming_product_tool_call(): @pytest.fixture -def qwen_tokenizer() -> AnyTokenizer: +def qwen_tokenizer() -> TokenizerLike: from vllm.transformers_utils.tokenizer import get_tokenizer return get_tokenizer("Qwen/Qwen3-32B") @pytest.fixture -def hermes_parser(qwen_tokenizer: AnyTokenizer) -> Hermes2ProToolParser: +def hermes_parser(qwen_tokenizer: TokenizerLike) -> Hermes2ProToolParser: return Hermes2ProToolParser(qwen_tokenizer) @@ -291,7 +291,7 @@ def any_chat_request() -> ChatCompletionRequest: def test_hermes_parser_streaming_just_forward_text( - qwen_tokenizer: AnyTokenizer, + qwen_tokenizer: TokenizerLike, hermes_parser: Hermes2ProToolParser, any_chat_request: ChatCompletionRequest, ) -> None: @@ -323,7 +323,7 @@ def test_hermes_parser_streaming_just_forward_text( def test_hermes_parser_streaming_failure_case_bug_19056( - qwen_tokenizer: AnyTokenizer, + qwen_tokenizer: TokenizerLike, hermes_parser: Hermes2ProToolParser, any_chat_request: ChatCompletionRequest, ) -> None: @@ -357,7 +357,7 @@ def test_hermes_parser_streaming_failure_case_bug_19056( def test_hermes_parser_streaming( - qwen_tokenizer: AnyTokenizer, + qwen_tokenizer: TokenizerLike, hermes_parser: Hermes2ProToolParser, any_chat_request: ChatCompletionRequest, ) -> None: diff --git a/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py index 37e52d2cdf60..bd563de438cb 100644 --- a/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py @@ -7,11 +7,11 @@ from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation from vllm.entrypoints.openai.tool_parsers.llama_tool_parser import Llama3JsonToolParser -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike @pytest.fixture -def parser(default_tokenizer: AnyTokenizer): +def parser(default_tokenizer: TokenizerLike): return Llama3JsonToolParser(default_tokenizer) diff --git a/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py index d297432eab64..5b67042264d4 100644 --- a/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike # Test cases similar to pythonic parser but with Llama4 specific format SIMPLE_FUNCTION_OUTPUT = "[get_weather(city='LA', metric='C')]" @@ -64,7 +64,7 @@ @pytest.mark.parametrize("streaming", [True, False]) -def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer): +def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike): tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( default_tokenizer ) @@ -208,7 +208,7 @@ def test_tool_call( streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall], - default_tokenizer: AnyTokenizer, + default_tokenizer: TokenizerLike, ): tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( default_tokenizer @@ -224,7 +224,7 @@ def test_tool_call( assert actual.function == expected -def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): +def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike): tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( default_tokenizer ) @@ -246,7 +246,7 @@ def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): @pytest.mark.parametrize("streaming", [False]) -def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer): +def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike): """test regex timeout is handled gracefully""" tool_parser: ToolParser = ToolParserManager.get_tool_parser("llama4_pythonic")( default_tokenizer diff --git a/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py index 13cff9a8ebf1..3e559702b838 100644 --- a/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')" @@ -69,7 +69,7 @@ @pytest.mark.parametrize("streaming", [True, False]) -def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer): +def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike): tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( default_tokenizer ) @@ -188,7 +188,7 @@ def test_tool_call( streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall], - default_tokenizer: AnyTokenizer, + default_tokenizer: TokenizerLike, ): tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( default_tokenizer @@ -205,7 +205,7 @@ def test_tool_call( assert actual.function == expected -def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): +def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike): tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( default_tokenizer ) @@ -228,7 +228,7 @@ def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): @pytest.mark.parametrize("streaming", [False]) -def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer): +def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike): """test regex timeout is handled gracefully""" tool_parser: ToolParser = ToolParserManager.get_tool_parser("olmo3")( default_tokenizer diff --git a/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py index fcd3df16e5cf..0382bc5bf035 100644 --- a/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')" @@ -61,7 +61,7 @@ @pytest.mark.parametrize("streaming", [True, False]) -def test_no_tool_call(streaming: bool, default_tokenizer: AnyTokenizer): +def test_no_tool_call(streaming: bool, default_tokenizer: TokenizerLike): tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( default_tokenizer ) @@ -168,7 +168,7 @@ def test_tool_call( streaming: bool, model_output: str, expected_tool_calls: list[FunctionCall], - default_tokenizer: AnyTokenizer, + default_tokenizer: TokenizerLike, ): tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( default_tokenizer @@ -185,7 +185,7 @@ def test_tool_call( assert actual.function == expected -def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): +def test_streaming_tool_call_with_large_steps(default_tokenizer: TokenizerLike): tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( default_tokenizer ) @@ -208,7 +208,7 @@ def test_streaming_tool_call_with_large_steps(default_tokenizer: AnyTokenizer): @pytest.mark.parametrize("streaming", [False]) -def test_regex_timeout_handling(streaming: bool, default_tokenizer: AnyTokenizer): +def test_regex_timeout_handling(streaming: bool, default_tokenizer: TokenizerLike): """test regex timeout is handled gracefully""" tool_parser: ToolParser = ToolParserManager.get_tool_parser("pythonic")( default_tokenizer diff --git a/tests/entrypoints/openai/tool_parsers/utils.py b/tests/entrypoints/openai/tool_parsers/utils.py index 38899f263255..3ab77e986eac 100644 --- a/tests/entrypoints/openai/tool_parsers/utils.py +++ b/tests/entrypoints/openai/tool_parsers/utils.py @@ -11,7 +11,7 @@ ToolCall, ) from vllm.entrypoints.openai.tool_parsers import ToolParser -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike class StreamingToolReconstructor: @@ -111,7 +111,7 @@ def run_tool_extraction_nonstreaming( return tool_parser.extract_tool_calls(model_output, request) -def split_string_into_token_deltas(tokenizer: AnyTokenizer, text: str) -> list[str]: +def split_string_into_token_deltas(tokenizer: TokenizerLike, text: str) -> list[str]: # Split a string into a series of deltas using the provided tokenizer. Each # delta will be the string equivalent of a single token. token_ids = tokenizer.encode(text, add_special_tokens=False) diff --git a/tests/models/multimodal/generation/vlm_utils/core.py b/tests/models/multimodal/generation/vlm_utils/core.py index 03ff3bcf6307..dfa109e5b605 100644 --- a/tests/models/multimodal/generation/vlm_utils/core.py +++ b/tests/models/multimodal/generation/vlm_utils/core.py @@ -9,7 +9,7 @@ from transformers.models.auto.auto_factory import _BaseAutoModelClass from vllm.config.model import RunnerOption -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from .....conftest import HfRunner, VllmRunner from ....registry import HF_EXAMPLE_MODELS @@ -33,7 +33,7 @@ def run_test( auto_cls: type[_BaseAutoModelClass], use_tokenizer_eos: bool, comparator: Callable[..., None], - get_stop_token_ids: Callable[[AnyTokenizer], list[int]] | None, + get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None, stop_str: list[str] | None, limit_mm_per_prompt: dict[str, int], vllm_runner_kwargs: dict[str, Any] | None, diff --git a/tests/models/multimodal/generation/vlm_utils/types.py b/tests/models/multimodal/generation/vlm_utils/types.py index 5c1bc6ac28fe..4dd96b142143 100644 --- a/tests/models/multimodal/generation/vlm_utils/types.py +++ b/tests/models/multimodal/generation/vlm_utils/types.py @@ -14,7 +14,7 @@ from vllm.config.model import RunnerOption from vllm.logprobs import SampleLogprobs -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from .....conftest import ( AUDIO_ASSETS, @@ -126,7 +126,7 @@ class VLMTestInfo(NamedTuple): vllm_runner_kwargs: dict[str, Any] | None = None # Optional callable which gets a list of token IDs from the model tokenizer - get_stop_token_ids: Callable[[AnyTokenizer], list[int]] | None = None + get_stop_token_ids: Callable[[TokenizerLike], list[int]] | None = None # Optional list of strings to stop generation, useful when stop tokens are # not special tokens in the tokenizer stop_str: list[str] | None = None diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 2f04bc6695c8..86cae9abe048 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -22,7 +22,7 @@ replace_token_matches, ) from vllm.multimodal.profiling import MultiModalProfiler -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from .utils import random_image @@ -237,7 +237,7 @@ def test_find_token_matches( update_type, ): # Should not be used since there is nothing to convert to token IDs - mock_tokenizer = cast(AnyTokenizer, object()) + mock_tokenizer = cast(TokenizerLike, object()) prompt_updates = { key: update_type(key, target, []).resolve(0) @@ -384,7 +384,7 @@ def test_find_text_matches( update_type, ): # Should not be used since there is nothing to convert to text - mock_tokenizer = cast(AnyTokenizer, object()) + mock_tokenizer = cast(TokenizerLike, object()) prompt_updates = { key: update_type(key, target, []).resolve(0) @@ -544,7 +544,7 @@ def test_find_update_text( expected_by_update_type_mm_count, ): # Should not be used since there is nothing to convert to text - mock_tokenizer = cast(AnyTokenizer, object()) + mock_tokenizer = cast(TokenizerLike, object()) for ( update_type, @@ -749,7 +749,7 @@ def test_find_update_tokens( expected_by_update_type_mm_count, ): # Should not be used since there is nothing to convert to tokens - mock_tokenizer = cast(AnyTokenizer, object()) + mock_tokenizer = cast(TokenizerLike, object()) for ( update_type, @@ -899,7 +899,7 @@ def test_find_mm_placeholders( update_type, ): # Should not be used since there is nothing to convert to tokens - mock_tokenizer = cast(AnyTokenizer, object()) + mock_tokenizer = cast(TokenizerLike, object()) mm_prompt_updates = { key: [[update_type(key, [], repl).resolve(i)] for i in range(3)] @@ -1028,7 +1028,7 @@ def test_hf_processor_init_kwargs( expected_kwargs, ): # Should not be used since there is nothing to convert to tokens - mock_tokenizer = cast(AnyTokenizer, object()) + mock_tokenizer = cast(TokenizerLike, object()) ctx = InputProcessingContext( model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs), @@ -1064,7 +1064,7 @@ def test_hf_processor_call_kwargs( expected_kwargs, ): # Should not be used since there is nothing to convert to tokens - mock_tokenizer = cast(AnyTokenizer, object()) + mock_tokenizer = cast(TokenizerLike, object()) ctx = InputProcessingContext( model_config=ModelConfig(model_id, mm_processor_kwargs=config_kwargs), diff --git a/tests/tokenization/__init__.py b/tests/tokenization/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/tokenization/test_do_lower_case.py b/tests/tokenization/test_do_lower_case.py deleted file mode 100644 index 8aff50b351e3..000000000000 --- a/tests/tokenization/test_do_lower_case.py +++ /dev/null @@ -1,18 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest - -from vllm.transformers_utils.tokenizer import get_tokenizer - -TOKENIZER_NAMES = ["BAAI/bge-base-en"] - - -@pytest.mark.parametrize("tokenizer_name", TOKENIZER_NAMES) -@pytest.mark.parametrize("n_tokens", [510]) -def test_special_tokens(tokenizer_name: str, n_tokens: int): - tokenizer = get_tokenizer(tokenizer_name, revision="main") - - prompts = "[UNK]" * n_tokens - prompt_token_ids = tokenizer.encode(prompts) - assert len(prompt_token_ids) == n_tokens + 2 diff --git a/tests/tokenization/test_tokenizer.py b/tests/tokenization/test_tokenizer.py deleted file mode 100644 index e86bb03883b5..000000000000 --- a/tests/tokenization/test_tokenizer.py +++ /dev/null @@ -1,23 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -import pytest -from transformers import PreTrainedTokenizerBase - -from vllm.transformers_utils.tokenizer import get_tokenizer - -TOKENIZER_NAMES = [ - "facebook/opt-125m", - "gpt2", -] - - -@pytest.mark.parametrize("tokenizer_name", TOKENIZER_NAMES) -def test_tokenizer_revision(tokenizer_name: str): - # Assume that "main" branch always exists - tokenizer = get_tokenizer(tokenizer_name, revision="main") - assert isinstance(tokenizer, PreTrainedTokenizerBase) - - # Assume that "never" branch always does not exist - with pytest.raises(OSError, match="not a valid git identifier"): - get_tokenizer(tokenizer_name, revision="never") diff --git a/tests/tokenization/test_tokenizer_registry.py b/tests/tokenization/test_tokenizer_registry.py deleted file mode 100644 index d89737888aa2..000000000000 --- a/tests/tokenization/test_tokenizer_registry.py +++ /dev/null @@ -1,124 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -from typing import TYPE_CHECKING, Any - -from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizer_base import TokenizerBase, TokenizerRegistry - -if TYPE_CHECKING: - from vllm.entrypoints.chat_utils import ChatCompletionMessageParam - - -class TestTokenizer(TokenizerBase): - @classmethod - def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer": - return TestTokenizer() - - @property - def all_special_tokens_extended(self) -> list[str]: - raise NotImplementedError() - - @property - def all_special_tokens(self) -> list[str]: - raise NotImplementedError() - - @property - def all_special_ids(self) -> list[int]: - raise NotImplementedError() - - @property - def bos_token_id(self) -> int: - return 0 - - @property - def eos_token_id(self) -> int: - return 1 - - @property - def sep_token(self) -> str: - raise NotImplementedError() - - @property - def pad_token(self) -> str: - raise NotImplementedError() - - @property - def is_fast(self) -> bool: - raise NotImplementedError() - - @property - def vocab_size(self) -> int: - raise NotImplementedError() - - @property - def max_token_id(self) -> int: - raise NotImplementedError() - - @property - def truncation_side(self) -> str: - raise NotImplementedError() - - def __call__( - self, - text: str | list[str] | list[int], - text_pair: str | None = None, - add_special_tokens: bool = False, - truncation: bool = False, - max_length: int | None = None, - ): - raise NotImplementedError() - - def get_vocab(self) -> dict[str, int]: - raise NotImplementedError() - - def get_added_vocab(self) -> dict[str, int]: - raise NotImplementedError() - - def encode_one( - self, - text: str, - truncation: bool = False, - max_length: int | None = None, - ) -> list[int]: - raise NotImplementedError() - - def encode(self, text: str, add_special_tokens: bool | None = None) -> list[int]: - raise NotImplementedError() - - def apply_chat_template( - self, - messages: list["ChatCompletionMessageParam"], - tools: list[dict[str, Any]] | None = None, - **kwargs, - ) -> list[int]: - raise NotImplementedError() - - def convert_tokens_to_string(self, tokens: list[str]) -> str: - raise NotImplementedError() - - def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str: - raise NotImplementedError() - - def convert_ids_to_tokens( - self, - ids: list[int], - skip_special_tokens: bool = True, - ) -> list[str]: - raise NotImplementedError() - - -def test_customized_tokenizer(): - TokenizerRegistry.register( - "test_tokenizer", "tests.tokenization.test_tokenizer_registry", "TestTokenizer" - ) - - tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer") - assert isinstance(tokenizer, TestTokenizer) - assert tokenizer.bos_token_id == 0 - assert tokenizer.eos_token_id == 1 - - tokenizer = get_tokenizer("test_tokenizer", tokenizer_mode="custom") - assert isinstance(tokenizer, TestTokenizer) - assert tokenizer.bos_token_id == 0 - assert tokenizer.eos_token_id == 1 diff --git a/tests/tool_use/test_ernie45_moe_tool_parser.py b/tests/tool_use/test_ernie45_moe_tool_parser.py index 36a07bb561d9..f199dcebe2f8 100644 --- a/tests/tool_use/test_ernie45_moe_tool_parser.py +++ b/tests/tool_use/test_ernie45_moe_tool_parser.py @@ -15,7 +15,7 @@ ) from vllm.entrypoints.openai.tool_parsers.ernie45_tool_parser import Ernie45ToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer # Use a common model that is likely to be available MODEL = "baidu/ERNIE-4.5-21B-A3B-Thinking" @@ -173,7 +173,7 @@ def test_extract_tool_calls( def stream_delta_message_generator( ernie45_tool_parser: Ernie45ToolParser, - ernie45_tokenizer: AnyTokenizer, + ernie45_tokenizer: TokenizerLike, model_output: str, request: ChatCompletionRequest | None = None, ) -> Generator[DeltaMessage, None, None]: diff --git a/tests/tool_use/test_jamba_tool_parser.py b/tests/tool_use/test_jamba_tool_parser.py index 9eb73b80fa9b..7ae9fc33d99a 100644 --- a/tests/tool_use/test_jamba_tool_parser.py +++ b/tests/tool_use/test_jamba_tool_parser.py @@ -11,7 +11,7 @@ from vllm.entrypoints.openai.protocol import DeltaMessage, FunctionCall, ToolCall from vllm.entrypoints.openai.tool_parsers.jamba_tool_parser import JambaToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer pytestmark = pytest.mark.cpu_test @@ -44,7 +44,9 @@ def assert_tool_calls( def stream_delta_message_generator( - jamba_tool_parser: JambaToolParser, jamba_tokenizer: AnyTokenizer, model_output: str + jamba_tool_parser: JambaToolParser, + jamba_tokenizer: TokenizerLike, + model_output: str, ) -> Generator[DeltaMessage, None, None]: all_token_ids = jamba_tokenizer.encode(model_output, add_special_tokens=False) diff --git a/tests/tool_use/test_qwen3coder_tool_parser.py b/tests/tool_use/test_qwen3coder_tool_parser.py index 93ef1049fc07..a18a0c37909f 100644 --- a/tests/tool_use/test_qwen3coder_tool_parser.py +++ b/tests/tool_use/test_qwen3coder_tool_parser.py @@ -18,7 +18,7 @@ ) from vllm.entrypoints.openai.tool_parsers.qwen3xml_tool_parser import Qwen3XMLToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer pytestmark = pytest.mark.cpu_test @@ -104,7 +104,7 @@ def assert_tool_calls( def stream_delta_message_generator( qwen3_tool_parser, - qwen3_tokenizer: AnyTokenizer, + qwen3_tokenizer: TokenizerLike, model_output: str, request: ChatCompletionRequest | None = None, ) -> Generator[DeltaMessage, None, None]: diff --git a/tests/tool_use/test_seed_oss_tool_parser.py b/tests/tool_use/test_seed_oss_tool_parser.py index 1367ad87cb01..fdcbfc75046c 100644 --- a/tests/tool_use/test_seed_oss_tool_parser.py +++ b/tests/tool_use/test_seed_oss_tool_parser.py @@ -16,7 +16,7 @@ ) from vllm.entrypoints.openai.tool_parsers.seed_oss_tool_parser import SeedOssToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer pytestmark = pytest.mark.cpu_test @@ -256,7 +256,7 @@ def test_streaming_tool_calls_no_tools(seed_oss_tool_parser): def stream_delta_message_generator( seed_oss_tool_parser: SeedOssToolParser, - seed_oss_tokenizer: AnyTokenizer, + seed_oss_tokenizer: TokenizerLike, model_output: str, request: ChatCompletionRequest | None = None, ) -> Generator[DeltaMessage, None, None]: diff --git a/tests/tool_use/test_xlam_tool_parser.py b/tests/tool_use/test_xlam_tool_parser.py index 122b427d6040..575f7f041d5a 100644 --- a/tests/tool_use/test_xlam_tool_parser.py +++ b/tests/tool_use/test_xlam_tool_parser.py @@ -14,7 +14,7 @@ ) from vllm.entrypoints.openai.tool_parsers.xlam_tool_parser import xLAMToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_tokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer pytestmark = pytest.mark.cpu_test @@ -49,7 +49,7 @@ def assert_tool_calls( def stream_delta_message_generator( xlam_tool_parser: xLAMToolParser, - xlam_tokenizer: AnyTokenizer, + xlam_tokenizer: TokenizerLike, model_output: str, request: ChatCompletionRequest | None = None, ) -> Generator[DeltaMessage, None, None]: diff --git a/tests/tokenization/test_cached_tokenizer.py b/tests/transformers_utils/test_cached_tokenizer.py similarity index 85% rename from tests/tokenization/test_cached_tokenizer.py rename to tests/transformers_utils/test_cached_tokenizer.py index 074039f9e513..308faa5a4430 100644 --- a/tests/tokenization/test_cached_tokenizer.py +++ b/tests/transformers_utils/test_cached_tokenizer.py @@ -6,7 +6,7 @@ import pytest from transformers import AutoTokenizer -from vllm.transformers_utils.tokenizer import AnyTokenizer, get_cached_tokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike, get_cached_tokenizer @pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"]) @@ -25,13 +25,12 @@ def test_cached_tokenizer(model_id: str): _check_consistency(unpickled_tokenizer, reference_tokenizer) -def _check_consistency(target: AnyTokenizer, expected: AnyTokenizer): +def _check_consistency(target: TokenizerLike, expected: TokenizerLike): assert isinstance(target, type(expected)) # Cached attributes assert target.all_special_ids == expected.all_special_ids assert target.all_special_tokens == expected.all_special_tokens - assert target.all_special_tokens_extended == expected.all_special_tokens_extended assert target.get_vocab() == expected.get_vocab() assert len(target) == len(expected) diff --git a/tests/tokenization/test_detokenize.py b/tests/transformers_utils/test_detokenize.py similarity index 100% rename from tests/tokenization/test_detokenize.py rename to tests/transformers_utils/test_detokenize.py diff --git a/tests/tokenization/test_get_eos.py b/tests/transformers_utils/test_get_eos.py similarity index 100% rename from tests/tokenization/test_get_eos.py rename to tests/transformers_utils/test_get_eos.py diff --git a/tests/tokenization/test_mistral_tokenizer.py b/tests/transformers_utils/test_mistral_tokenizer.py similarity index 98% rename from tests/tokenization/test_mistral_tokenizer.py rename to tests/transformers_utils/test_mistral_tokenizer.py index c80b698ba384..c3e04eaa7993 100644 --- a/tests/tokenization/test_mistral_tokenizer.py +++ b/tests/transformers_utils/test_mistral_tokenizer.py @@ -260,7 +260,6 @@ class TestMistralTokenizer: def test_all_special_tokens(self, mistral_tokenizer: MistralTokenizer): attributes = [ mistral_tokenizer.all_special_tokens, - mistral_tokenizer.all_special_tokens_extended, ] for attribute in attributes: @@ -314,25 +313,6 @@ def get_vocab(self, mistral_tokenizer: MistralTokenizer): def test_get_added_vocab(self, mistral_tokenizer: MistralTokenizer): assert mistral_tokenizer.get_added_vocab() == {} - def test_encode_one(self, mistral_tokenizer: MistralTokenizer): - token_ids = ( - [22177, 4304, 2662] if mistral_tokenizer.is_tekken else [23325, 2294, 1686] - ) - - assert mistral_tokenizer.encode_one("Hello world !") == token_ids - assert mistral_tokenizer.encode_one("Hello world !", max_length=1) == token_ids - assert ( - mistral_tokenizer.encode_one("Hello world !", truncation=True, max_length=1) - == token_ids[:-2] - ) - assert ( - mistral_tokenizer.encode_one( - "Hello world !", truncation=False, max_length=1 - ) - == token_ids - ) - assert mistral_tokenizer.encode_one("") == [] - def test_encode(self, mistral_tokenizer: MistralTokenizer): token_ids = ( [1, 22177, 4304, 2662] diff --git a/tests/transformers_utils/test_get_processor_kwargs_from_processor.py b/tests/transformers_utils/test_processor.py similarity index 100% rename from tests/transformers_utils/test_get_processor_kwargs_from_processor.py rename to tests/transformers_utils/test_processor.py diff --git a/tests/transformers_utils/test_config.py b/tests/transformers_utils/test_repo_utils.py similarity index 100% rename from tests/transformers_utils/test_config.py rename to tests/transformers_utils/test_repo_utils.py diff --git a/tests/transformers_utils/test_tokenizer.py b/tests/transformers_utils/test_tokenizer.py new file mode 100644 index 000000000000..5aa413548d74 --- /dev/null +++ b/tests/transformers_utils/test_tokenizer.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import _get_protocol_attrs # type: ignore + +import pytest +from transformers import PreTrainedTokenizerBase + +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizer_base import TokenizerLike + + +def _get_missing_attrs(obj: object, target: type): + return [k for k in _get_protocol_attrs(target) if not hasattr(obj, k)] + + +def test_tokenizer_like_protocol(): + assert isinstance( + tokenizer := get_tokenizer("gpt2", use_fast=False), + TokenizerLike, + ), f"Missing attrs: {_get_missing_attrs(tokenizer, TokenizerLike)}" + + assert isinstance( + tokenizer := get_tokenizer("gpt2", use_fast=True), + TokenizerLike, + ), f"Missing attrs: {_get_missing_attrs(tokenizer, TokenizerLike)}" + + assert isinstance( + tokenizer := get_tokenizer( + "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral" + ), + TokenizerLike, + ), f"Missing attrs: {_get_missing_attrs(tokenizer, TokenizerLike)}" + + +@pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"]) +def test_tokenizer_revision(tokenizer_name: str): + # Assume that "main" branch always exists + tokenizer = get_tokenizer(tokenizer_name, revision="main") + assert isinstance(tokenizer, PreTrainedTokenizerBase) + + # Assume that "never" branch always does not exist + with pytest.raises(OSError, match="not a valid git identifier"): + get_tokenizer(tokenizer_name, revision="never") + + +@pytest.mark.parametrize("tokenizer_name", ["BAAI/bge-base-en"]) +@pytest.mark.parametrize("n_tokens", [510]) +def test_special_tokens(tokenizer_name: str, n_tokens: int): + tokenizer = get_tokenizer(tokenizer_name, revision="main") + + prompts = "[UNK]" * n_tokens + prompt_token_ids = tokenizer.encode(prompts) + assert len(prompt_token_ids) == n_tokens + 2 diff --git a/tests/transformers_utils/test_tokenizer_registry.py b/tests/transformers_utils/test_tokenizer_registry.py new file mode 100644 index 000000000000..1ae0501177af --- /dev/null +++ b/tests/transformers_utils/test_tokenizer_registry.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizer_base import TokenizerLike, TokenizerRegistry + + +class TestTokenizer(TokenizerLike): + @classmethod + def from_pretrained(cls, *args, **kwargs) -> "TestTokenizer": + return TestTokenizer() # type: ignore + + @property + def bos_token_id(self) -> int: + return 0 + + @property + def eos_token_id(self) -> int: + return 1 + + +def test_customized_tokenizer(): + TokenizerRegistry.register( + "test_tokenizer", + "tests.transformers_utils.test_tokenizer_registry", + "TestTokenizer", + ) + + tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer") + assert isinstance(tokenizer, TestTokenizer) + assert tokenizer.bos_token_id == 0 + assert tokenizer.eos_token_id == 1 + + tokenizer = get_tokenizer("test_tokenizer", tokenizer_mode="custom") + assert isinstance(tokenizer, TestTokenizer) + assert tokenizer.bos_token_id == 0 + assert tokenizer.eos_token_id == 1 diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 8e1198b315bd..39d6bc385db2 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -18,7 +18,7 @@ from vllm.lora.request import LoRARequest from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import RequestOutputKind, SamplingParams -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.v1.engine import ( EngineCoreEvent, EngineCoreEventType, @@ -31,7 +31,7 @@ def _ref_convert_id_to_token( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, token_id: int, ) -> str: """Reference impl of logprobs detokenization. diff --git a/tools/pre_commit/check_pickle_imports.py b/tools/pre_commit/check_pickle_imports.py index b96a6701333d..0a12fe818670 100644 --- a/tools/pre_commit/check_pickle_imports.py +++ b/tools/pre_commit/check_pickle_imports.py @@ -28,7 +28,7 @@ "vllm/distributed/device_communicators/shm_object_storage.py", "vllm/utils/hashing.py", "tests/utils_/test_hashing.py", - "tests/tokenization/test_cached_tokenizer.py", + "tests/transformers_utils/test_cached_tokenizer.py", "benchmarks/kernels/graph_machete_bench.py", "benchmarks/kernels/benchmark_lora.py", "benchmarks/kernels/benchmark_machete.py", diff --git a/vllm/benchmarks/datasets.py b/vllm/benchmarks/datasets.py index 5411ecbb27b2..37b0497c974b 100644 --- a/vllm/benchmarks/datasets.py +++ b/vllm/benchmarks/datasets.py @@ -39,7 +39,7 @@ from vllm.lora.utils import get_adapter_absolute_path from vllm.multimodal import MultiModalDataDict from vllm.multimodal.image import convert_image_mode -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.import_utils import PlaceholderModule try: @@ -293,7 +293,7 @@ def lora_path_on_disk(lora_path: str) -> str: # Global cache for LoRA tokenizers. -lora_tokenizer_cache: dict[int, AnyTokenizer] = {} +lora_tokenizer_cache: dict[int, TokenizerLike] = {} def process_image(image: Any) -> Mapping[str, Any]: diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 6b3ee042daf3..5b3519866fc9 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -13,7 +13,7 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.processor import Processor @@ -85,7 +85,7 @@ async def abort(self, request_id: str | Iterable[str]) -> None: ... @abstractmethod - async def get_tokenizer(self) -> AnyTokenizer: + async def get_tokenizer(self) -> TokenizerLike: """Get the tokenizer""" ... diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index bf80856c1bbf..4ca99531d27d 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -51,7 +51,7 @@ from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path from vllm.transformers_utils.processor import cached_get_processor -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike from vllm.utils import random_uuid from vllm.utils.func_utils import supports_kw @@ -536,7 +536,7 @@ def resolve_hf_chat_template( def _resolve_chat_template_content_format( chat_template: str | None, tools: list[dict[str, Any]] | None, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, model_config: ModelConfig, ) -> _ChatTemplateContentFormat: @@ -593,7 +593,7 @@ def resolve_chat_template_content_format( chat_template: str | None, tools: list[dict[str, Any]] | None, given_format: ChatTemplateContentFormatOption, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, model_config: ModelConfig, ) -> _ChatTemplateContentFormat: @@ -627,7 +627,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]): maximum per prompt. """ - def __init__(self, model_config: ModelConfig, tokenizer: AnyTokenizer): + def __init__(self, model_config: ModelConfig, tokenizer: TokenizerLike): super().__init__() self._model_config = model_config @@ -1592,7 +1592,7 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None: def parse_chat_messages( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, content_format: _ChatTemplateContentFormat, ) -> tuple[ list[ConversationMessage], @@ -1624,7 +1624,7 @@ def parse_chat_messages( def parse_chat_messages_futures( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, content_format: _ChatTemplateContentFormat, ) -> tuple[ list[ConversationMessage], diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index f6ee74678998..07b1f319d40e 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -72,8 +72,8 @@ from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams from vllm.tasks import PoolingTask from vllm.transformers_utils.tokenizer import ( - AnyTokenizer, MistralTokenizer, + TokenizerLike, get_cached_tokenizer, ) from vllm.usage.usage_lib import UsageContext @@ -350,11 +350,11 @@ def __init__( self.processor = self.llm_engine.processor self.io_processor = self.llm_engine.io_processor - def get_tokenizer(self) -> AnyTokenizer: + def get_tokenizer(self) -> TokenizerLike: return self.llm_engine.get_tokenizer() @deprecated("`set_tokenizer` is deprecated and will be removed in v0.13.") - def set_tokenizer(self, tokenizer: AnyTokenizer) -> None: + def set_tokenizer(self, tokenizer: TokenizerLike) -> None: # While CachedTokenizer is dynamic, have no choice but # compare class name. Misjudgment will arise from # user-defined tokenizer started with 'Cached' @@ -1244,7 +1244,7 @@ def reward( def _embedding_score( self, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, text_1: list[str | TextPrompt | TokensPrompt], text_2: list[str | TextPrompt | TokensPrompt], truncate_prompt_tokens: int | None = None, @@ -1276,7 +1276,7 @@ def _embedding_score( def _cross_encoding_score( self, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, data_1: list[str] | list[ScoreContentPartParam], data_2: list[str] | list[ScoreContentPartParam], truncate_prompt_tokens: int | None = None, diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 9a7051e0920a..adf4dfbfd8d6 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -62,7 +62,7 @@ from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike from vllm.transformers_utils.tokenizers import ( maybe_serialize_tool_calls, truncate_tool_call_ids, @@ -530,7 +530,7 @@ async def chat_completion_stream_generator( request_id: str, model_name: str, conversation: list[ConversationMessage], - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, ) -> AsyncGenerator[str, None]: created_time = int(time.time()) @@ -1296,7 +1296,7 @@ async def chat_completion_full_generator( request_id: str, model_name: str, conversation: list[ConversationMessage], - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, ) -> ErrorResponse | ChatCompletionResponse: created_time = int(time.time()) @@ -1624,7 +1624,7 @@ def _get_top_logprobs( self, logprobs: dict[int, Logprob], top_logprobs: int | None, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, should_return_as_token_id: bool, ) -> list[ChatCompletionLogProb]: return [ @@ -1648,7 +1648,7 @@ def _create_chat_logprobs( self, token_ids: GenericSequence[int], top_logprobs: GenericSequence[dict[int, Logprob] | None], - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, num_output_top_logprobs: int | None = None, return_as_token_id: bool | None = None, ) -> ChatCompletionLogProbs: diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index 9681aa8c71e6..f86c46308ba2 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -33,7 +33,7 @@ from vllm.logprobs import Logprob from vllm.outputs import RequestOutput from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.async_utils import merge_async_iterators from vllm.utils.collection_utils import as_list from vllm.v1.sample.logits_processor import validate_logits_processors_parameters @@ -326,7 +326,7 @@ async def completion_stream_generator( created_time: int, model_name: str, num_prompts: int, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike | None, request_metadata: RequestResponseMetadata, ) -> AsyncGenerator[str, None]: num_choices = 1 if request.n is None else request.n @@ -511,7 +511,7 @@ def request_output_to_completion_response( request_id: str, created_time: int, model_name: str, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike | None, request_metadata: RequestResponseMetadata, ) -> CompletionResponse: choices: list[CompletionResponseChoice] = [] @@ -622,7 +622,7 @@ def _create_completion_logprobs( token_ids: GenericSequence[int], top_logprobs: GenericSequence[dict[int, Logprob] | None], num_output_top_logprobs: int, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike | None, initial_text_offset: int = 0, return_as_token_id: bool | None = None, ) -> CompletionLogProbs: @@ -642,9 +642,16 @@ def _create_completion_logprobs( for i, token_id in enumerate(token_ids): step_top_logprobs = top_logprobs[i] if step_top_logprobs is None: - token = tokenizer.decode(token_id) if should_return_as_token_id: token = f"token_id:{token_id}" + else: + if tokenizer is None: + raise ValueError( + "Unable to get tokenizer because " + "skip_tokenizer_init is True" + ) + + token = tokenizer.decode(token_id) out_tokens.append(token) out_token_logprobs.append(None) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index d9feee917ff4..ffcee78521c5 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -101,7 +101,7 @@ extract_trace_headers, log_tracing_disabled_warning, ) -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike from vllm.utils import random_uuid from vllm.utils.async_utils import ( AsyncMicrobatchTokenizer, @@ -227,7 +227,7 @@ class ServeContext( lora_request: LoRARequest | None = None # Shared across most requests - tokenizer: AnyTokenizer | None = None + tokenizer: TokenizerLike | None = None # `protected_namespaces` resolves Pydantic v2's warning # on conflict with protected namespace "model_" @@ -281,7 +281,7 @@ def __init__( apply_mistral_chat_template, executor=self._tokenizer_executor ) - self._async_tokenizer_pool: dict[AnyTokenizer, AsyncMicrobatchTokenizer] = {} + self._async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer] = {} self.log_error_stack = log_error_stack self.processor = self.models.processor @@ -291,7 +291,7 @@ def __init__( def _get_tool_parser( self, tool_parser_name: str | None = None, enable_auto_tools: bool = False - ) -> Callable[[AnyTokenizer], ToolParser] | None: + ) -> Callable[[TokenizerLike], ToolParser] | None: """Get the tool parser based on the name.""" parser = None if not enable_auto_tools or tool_parser_name is None: @@ -317,7 +317,7 @@ def _get_tool_parser( def _get_reasoning_parser( self, reasoning_parser_name: str, - ) -> Callable[[AnyTokenizer], ReasoningParser] | None: + ) -> Callable[[TokenizerLike], ReasoningParser] | None: """Get the reasoning parser based on the name.""" parser = None if not reasoning_parser_name: @@ -547,7 +547,7 @@ async def beam_search( prompt_logprobs=None, ) - def _get_renderer(self, tokenizer: AnyTokenizer | None) -> BaseRenderer: + def _get_renderer(self, tokenizer: TokenizerLike | None) -> BaseRenderer: """ Get a Renderer instance with the provided tokenizer. Uses shared async tokenizer pool for efficiency. @@ -877,7 +877,7 @@ async def _normalize_prompt_text_to_input( self, request: AnyRequest, prompt: str, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, add_special_tokens: bool, ) -> TextTokensPrompt: async_tokenizer = self._get_async_tokenizer(tokenizer) @@ -919,7 +919,7 @@ async def _normalize_prompt_tokens_to_input( self, request: AnyRequest, prompt_ids: list[int], - tokenizer: AnyTokenizer | None, + tokenizer: TokenizerLike | None, ) -> TextTokensPrompt: truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None) @@ -1015,7 +1015,7 @@ def _validate_input( async def _tokenize_prompt_input_async( self, request: AnyRequest, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, prompt_input: str | list[int], add_special_tokens: bool = True, ) -> TextTokensPrompt: @@ -1034,7 +1034,7 @@ async def _tokenize_prompt_input_async( async def _tokenize_prompt_inputs_async( self, request: AnyRequest, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, prompt_inputs: Iterable[str | list[int]], add_special_tokens: bool = True, ) -> AsyncGenerator[TextTokensPrompt, None]: @@ -1079,7 +1079,7 @@ def _validate_chat_template( async def _preprocess_chat( self, request: ChatLikeRequest | ResponsesRequest, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, messages: list[ChatCompletionMessageParam], chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, @@ -1088,7 +1088,7 @@ async def _preprocess_chat( tool_dicts: list[dict[str, Any]] | None = None, documents: list[dict[str, str]] | None = None, chat_template_kwargs: dict[str, Any] | None = None, - tool_parser: Callable[[AnyTokenizer], ToolParser] | None = None, + tool_parser: Callable[[TokenizerLike], ToolParser] | None = None, add_special_tokens: bool = False, ) -> tuple[ list[ConversationMessage], @@ -1370,9 +1370,9 @@ def _get_data_parallel_rank(raw_request: Request | None) -> int | None: @staticmethod def _parse_tool_calls_from_content( request: ResponsesRequest | ChatCompletionRequest, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, enable_auto_tools: bool, - tool_parser_cls: Callable[[AnyTokenizer], ToolParser] | None, + tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None, content: str | None = None, ) -> tuple[list[FunctionCall] | None, str | None]: function_calls = list[FunctionCall]() @@ -1442,7 +1442,7 @@ def _parse_tool_calls_from_content( def _get_decoded_token( logprob: Logprob, token_id: int, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike | None, return_as_token_id: bool = False, ) -> str: if return_as_token_id: @@ -1450,6 +1450,12 @@ def _get_decoded_token( if logprob.decoded_token is not None: return logprob.decoded_token + + if tokenizer is None: + raise ValueError( + "Unable to get tokenizer because skip_tokenizer_init is True" + ) + return tokenizer.decode(token_id) def _is_model_supported(self, model_name: str | None) -> bool: diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index f546dbda7fef..103df2369f59 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -105,7 +105,7 @@ from vllm.logprobs import SampleLogprobs from vllm.outputs import CompletionOutput from vllm.sampling_params import SamplingParams, StructuredOutputsParams -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) @@ -492,7 +492,7 @@ async def _make_request( self, request: ResponsesRequest, prev_response: ResponsesResponse | None, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ): if request.tools is None or ( request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none @@ -563,7 +563,7 @@ async def responses_full_generator( result_generator: AsyncIterator[ConversationContext], context: ConversationContext, model_name: str, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, created_time: int | None = None, ) -> ErrorResponse | ResponsesResponse: @@ -675,7 +675,7 @@ def _topk_logprobs( self, logprobs: dict[int, SampleLogprob], top_logprobs: int, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> list[LogprobTopLogprob]: """Returns the top-k logprobs from the logprobs dictionary.""" out = [] @@ -700,7 +700,7 @@ def _create_response_logprobs( self, token_ids: Sequence[int], logprobs: SampleLogprobs | None, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, top_logprobs: int | None = None, ) -> list[Logprob]: assert logprobs is not None, "logprobs must be provided" @@ -736,7 +736,7 @@ def _create_stream_response_logprobs( self, token_ids: Sequence[int], logprobs: SampleLogprobs | None, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, top_logprobs: int | None = None, ) -> list[response_text_delta_event.Logprob]: lgs = self._create_response_logprobs( @@ -763,7 +763,7 @@ def _make_response_output_items( self, request: ResponsesRequest, final_output: CompletionOutput, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> list[ResponseOutputItem]: if self.reasoning_parser: try: @@ -1135,7 +1135,7 @@ async def _process_simple_streaming_events( result_generator: AsyncIterator[ConversationContext | None], context: ConversationContext, model_name: str, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, created_time: int, _increment_sequence_number_and_return: Callable[ @@ -1438,7 +1438,7 @@ async def _process_harmony_streaming_events( result_generator: AsyncIterator[ConversationContext | None], context: ConversationContext, model_name: str, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, created_time: int, _increment_sequence_number_and_return: Callable[ @@ -1891,7 +1891,7 @@ async def responses_stream_generator( result_generator: AsyncIterator[ConversationContext | None], context: ConversationContext, model_name: str, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, request_metadata: RequestResponseMetadata, created_time: int | None = None, ) -> AsyncGenerator[StreamingResponsesResponse, None]: diff --git a/vllm/entrypoints/openai/serving_score.py b/vllm/entrypoints/openai/serving_score.py index 9cbfc9791819..173815093316 100644 --- a/vllm/entrypoints/openai/serving_score.py +++ b/vllm/entrypoints/openai/serving_score.py @@ -36,7 +36,7 @@ from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike from vllm.utils.async_utils import make_async, merge_async_iterators logger = init_logger(__name__) @@ -60,7 +60,7 @@ def __init__( async def _embedding_score( self, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, texts_1: list[str], texts_2: list[str], request: RerankRequest | ScoreRequest, @@ -153,7 +153,7 @@ async def _embedding_score( def _preprocess_score( self, request: RerankRequest | ScoreRequest, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, tokenization_kwargs: dict[str, Any], data_1: str | ScoreContentPartParam, data_2: str | ScoreContentPartParam, @@ -175,7 +175,7 @@ def _preprocess_score( async def _cross_encoding_score( self, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, data_1: list[str] | list[ScoreContentPartParam], data_2: list[str] | list[ScoreContentPartParam], request: RerankRequest | ScoreRequest, diff --git a/vllm/entrypoints/openai/serving_tokenization.py b/vllm/entrypoints/openai/serving_tokenization.py index 39aae0cd0495..a077e4d62fab 100644 --- a/vllm/entrypoints/openai/serving_tokenization.py +++ b/vllm/entrypoints/openai/serving_tokenization.py @@ -22,7 +22,7 @@ from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.renderer import RenderConfig from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) @@ -170,7 +170,7 @@ def _build_render_config(self, request: TokenizeRequest) -> RenderConfig: @dataclass class TokenizerInfo: - tokenizer: AnyTokenizer + tokenizer: TokenizerLike chat_template: str | None def to_dict(self) -> dict[str, Any]: diff --git a/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py index e99e405f5de6..9c917fe7a3b3 100644 --- a/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py @@ -22,7 +22,7 @@ from vllm.sampling_params import ( StructuredOutputsParams, ) -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.collection_utils import is_list_of from vllm.utils.import_utils import import_from_path @@ -36,7 +36,7 @@ class ToolParser: derived classes. """ - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): self.prev_tool_call_arr: list[dict] = [] # the index of the tool call that is currently being parsed self.current_tool_id: int = -1 diff --git a/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py index cbeb879969ec..5833bd116b84 100644 --- a/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py @@ -19,13 +19,13 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class DeepSeekV31ToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.current_tool_name_sent: bool = False diff --git a/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py index bf7f6fa61ab9..b249a733cbbc 100644 --- a/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py @@ -19,13 +19,13 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class DeepSeekV3ToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.current_tool_name_sent: bool = False diff --git a/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py index 82370323cb00..56a769ed65c0 100644 --- a/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py @@ -19,13 +19,13 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class Ernie45ToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): """ Ernie thinking model format: abc\n\n\n\n\ndef\n\n diff --git a/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py index 389e9754b34d..828b4a26b3d5 100644 --- a/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py @@ -22,13 +22,13 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class Glm4MoeModelToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.current_tool_name_sent = False self.prev_tool_call_arr: list[dict] = [] diff --git a/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py index ae9217426fb5..bcc7ec1f86fb 100644 --- a/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py @@ -29,7 +29,7 @@ partial_json_loads, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) @@ -44,7 +44,7 @@ class Granite20bFCToolParser(ToolParser): are all set """ - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.bot_token = "" diff --git a/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py index d29c427694dc..392f40a0f3b6 100644 --- a/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py @@ -27,7 +27,7 @@ partial_json_loads, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) @@ -42,7 +42,7 @@ class GraniteToolParser(ToolParser): are all set """ - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) # for granite 3.0, the token `<|tool_call|>` self.bot_token = "<|tool_call|>" diff --git a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py index 4336a5438109..03a807c40f4e 100644 --- a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py @@ -22,13 +22,13 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike logger = init_logger(__name__) class Hermes2ProToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) if isinstance(self.model_tokenizer, MistralTokenizer): diff --git a/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py index 920675c8389b..82f149bc0ce8 100644 --- a/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py @@ -22,14 +22,14 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import consume_space from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) class HunyuanA13BToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) # Initialize state for streaming mode diff --git a/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py index 1dd327f645b3..b2f73a7a5116 100644 --- a/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py @@ -22,13 +22,13 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class Internlm2ToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.position = 0 diff --git a/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py index 6f53ddea4f0e..71ce3432835f 100644 --- a/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py @@ -21,14 +21,14 @@ from vllm.entrypoints.openai.tool_parsers import ToolParser from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.transformers_utils.tokenizers import MistralTokenizer logger = init_logger(__name__) class JambaToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) if isinstance(self.model_tokenizer, MistralTokenizer): diff --git a/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py index 2b84c60a3b84..5933cff3f550 100644 --- a/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py @@ -19,13 +19,13 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class KimiK2ToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.current_tool_name_sent: bool = False self.prev_tool_call_arr: list[dict] = [] diff --git a/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py index c6c8ae8ae95f..57a790a59516 100644 --- a/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py @@ -4,11 +4,11 @@ import regex as re from vllm.entrypoints.openai.tool_parsers.hermes_tool_parser import Hermes2ProToolParser -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike class LongcatFlashToolParser(Hermes2ProToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.tool_call_start_token: str = "" diff --git a/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py index 5c2258ba62b2..6117081d332e 100644 --- a/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py @@ -21,13 +21,13 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class MinimaxM2ToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.prev_tool_call_arr: list[dict] = [] diff --git a/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py index 982518a52e3d..9fb43e66b56d 100644 --- a/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py @@ -22,13 +22,13 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class MinimaxToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) # Initialize streaming state for tracking tool call progress diff --git a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py index 85671271522d..8d7b57423469 100644 --- a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py @@ -25,7 +25,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer +from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike logger = init_logger(__name__) @@ -46,7 +46,7 @@ def is_valid_id(id: str) -> bool: return id.isalnum() and len(id) == 9 -def _is_fn_name_regex_support(model_tokenizer: AnyTokenizer) -> bool: +def _is_fn_name_regex_support(model_tokenizer: TokenizerLike) -> bool: return ( isinstance(model_tokenizer, MistralTokenizer) and model_tokenizer.version >= 11 ) @@ -61,7 +61,7 @@ class MistralToolParser(ToolParser): Used when --enable-auto-tool-choice --tool-call-parser mistral are all set """ - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) if not isinstance(self.model_tokenizer, MistralTokenizer): diff --git a/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py index d1b36a297e0b..03eda6f1a539 100644 --- a/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py @@ -18,15 +18,15 @@ from vllm.logger import init_logger if TYPE_CHECKING: - from vllm.transformers_utils.tokenizer import AnyTokenizer + from vllm.transformers_utils.tokenizer import TokenizerLike else: - AnyTokenizer = object + TokenizerLike = object logger = init_logger(__name__) class OpenAIToolParser(ToolParser): - def __init__(self, tokenizer: "AnyTokenizer"): + def __init__(self, tokenizer: "TokenizerLike"): super().__init__(tokenizer) def extract_tool_calls( diff --git a/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py index 9d4c079eba18..5c054819322a 100644 --- a/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py @@ -22,13 +22,13 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) class Qwen3CoderToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.current_tool_name_sent: bool = False diff --git a/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py index 432c419db189..ccfffc91b481 100644 --- a/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py @@ -23,7 +23,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) @@ -1165,7 +1165,7 @@ def _reset_xml_parser_after_tool_call(self): class Qwen3XMLToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.parser = StreamingXMLToolCallParser() diff --git a/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py index 8aed7f0e9fc9..bf15b69494e0 100644 --- a/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py @@ -25,7 +25,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) @@ -34,7 +34,7 @@ class SeedOssToolParser(ToolParser): TOOL_CALL_START = "" TOOL_CALL_END = "" - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) # --- streaming state --- diff --git a/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py index adcb9f476547..e5ea6dcaf007 100644 --- a/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py @@ -21,7 +21,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) @@ -41,7 +41,7 @@ class Step3ToolParser(ToolParser): TOOL_SEP = "<｜tool_sep｜>" SPECIAL_TOKENS = [TOOL_CALLS_BEGIN, TOOL_CALLS_END, TOOL_CALL_BEGIN, TOOL_CALL_END] - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) self.position = 0 # Explicit state flags for robust streaming diff --git a/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py index 9d308af4de60..d50d6937f6b8 100644 --- a/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py @@ -21,14 +21,14 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) class xLAMToolParser(ToolParser): - def __init__(self, tokenizer: AnyTokenizer): + def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) # Initialize state for streaming mode diff --git a/vllm/entrypoints/renderer.py b/vllm/entrypoints/renderer.py index 3c5a396a99f9..8f5a05c3707d 100644 --- a/vllm/entrypoints/renderer.py +++ b/vllm/entrypoints/renderer.py @@ -16,7 +16,7 @@ from vllm.inputs.data import TextPrompt as EngineTextPrompt from vllm.inputs.data import TokensPrompt as EngineTokensPrompt from vllm.inputs.parse import get_prompt_components, parse_raw_prompts -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.async_utils import AsyncMicrobatchTokenizer @@ -85,7 +85,7 @@ class BaseRenderer(ABC): def __init__( self, model_config: ModelConfig, - tokenizer: AnyTokenizer | None = None, + tokenizer: TokenizerLike | None = None, ): super().__init__() self.model_config = model_config @@ -200,8 +200,8 @@ class CompletionRenderer(BaseRenderer): def __init__( self, model_config: ModelConfig, - tokenizer: AnyTokenizer | None = None, - async_tokenizer_pool: dict[AnyTokenizer, AsyncMicrobatchTokenizer] + tokenizer: TokenizerLike | None = None, + async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer] | None = None, ): super().__init__(model_config, tokenizer) @@ -373,7 +373,7 @@ def _get_async_tokenizer(self) -> AsyncMicrobatchTokenizer: return async_tokenizer tokenizer = self.tokenizer - if self.tokenizer is None: + if tokenizer is None: raise ValueError("No tokenizer available for text input processing") if self.async_tokenizer_pool is None: diff --git a/vllm/entrypoints/score_utils.py b/vllm/entrypoints/score_utils.py index 309a4c996392..87f698efce24 100644 --- a/vllm/entrypoints/score_utils.py +++ b/vllm/entrypoints/score_utils.py @@ -20,9 +20,9 @@ from vllm.multimodal.inputs import MultiModalDataDict from vllm.outputs import PoolingRequestOutput from vllm.transformers_utils.tokenizer import ( - AnyTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast, + TokenizerLike, ) ScoreContentPartParam: TypeAlias = ( @@ -93,7 +93,7 @@ def parse_score_data( data_1: str | ScoreContentPartParam, data_2: str | ScoreContentPartParam, model_config: ModelConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> tuple[str, str, MultiModalDataDict | None]: mm_tracker = MultiModalItemTracker(model_config, tokenizer) @@ -181,7 +181,7 @@ def post_process_tokens( def get_score_prompt( model_config: ModelConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, tokenization_kwargs: dict[str, Any], data_1: str | ScoreContentPartParam, data_2: str | ScoreContentPartParam, diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py index 839c13868a16..47897083e5ff 100644 --- a/vllm/inputs/preprocess.py +++ b/vllm/inputs/preprocess.py @@ -17,7 +17,7 @@ MultiModalUUIDDict, ) from vllm.multimodal.processing import BaseMultiModalProcessor -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.jsontree import json_iter_leaves from vllm.v1.metrics.stats import MultiModalCacheStats @@ -46,7 +46,7 @@ class InputPreprocessor: def __init__( self, model_config: ModelConfig, - tokenizer: AnyTokenizer | None, + tokenizer: TokenizerLike | None, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, mm_processor_cache: BaseMultiModalProcessorCache | None = None, ) -> None: @@ -59,7 +59,7 @@ def __init__( self.mm_cache_stats = MultiModalCacheStats() if mm_processor_cache else None - def get_tokenizer(self) -> AnyTokenizer: + def get_tokenizer(self) -> TokenizerLike: if self.tokenizer is None: raise ValueError( "You cannot pass text prompts when `skip_tokenizer_init` is True" @@ -228,11 +228,11 @@ def _tokenize_prompt( return tokenizer.encode(prompt, **tokenization_kwargs) - def _get_mm_tokenizer(self) -> AnyTokenizer: + def _get_mm_tokenizer(self) -> TokenizerLike: # PrithviGeoSpatialMAE needs to be initialized without a tokenizer # while using also multi-modal input if not self.tokenizer: - return cast(AnyTokenizer, object()) # Dummy + return cast(TokenizerLike, object()) # Dummy tokenizer = self.get_tokenizer() return tokenizer diff --git a/vllm/logits_process.py b/vllm/logits_process.py index 7b6a6528e20e..b4f8c06a3a61 100644 --- a/vllm/logits_process.py +++ b/vllm/logits_process.py @@ -5,7 +5,7 @@ import torch -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike LogitsProcessor: TypeAlias = ( Callable[[list[int], torch.Tensor], torch.Tensor] @@ -19,7 +19,7 @@ def get_bad_words_logits_processors( - bad_words: list[str], tokenizer: AnyTokenizer + bad_words: list[str], tokenizer: TokenizerLike ) -> list[LogitsProcessor]: bad_words_ids: list[list[int]] = list() diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index 81c6b34bd6ce..cec04dc01134 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -28,7 +28,7 @@ PromptUpdate, PromptUpdateDetails, ) -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from .intern_vit import InternVisionModel from .internvl import ( @@ -241,7 +241,7 @@ class H2OVLProcessor(BaseInternVLProcessor): def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, min_dynamic_patch: int | None = None, max_dynamic_patch: int | None = None, diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index ccbde115009d..884c92076900 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -50,7 +50,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import set_default_torch_num_threads @@ -347,7 +347,7 @@ class BaseInternVLProcessor(ABC): def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, min_dynamic_patch: int | None = None, max_dynamic_patch: int | None = None, @@ -561,7 +561,7 @@ class InternVLProcessor(BaseInternVLProcessor): def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, min_dynamic_patch: int | None = None, max_dynamic_patch: int | None = None, diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 5529089e06ae..5177174a2d03 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -75,7 +75,7 @@ from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.radio import RadioConfig from vllm.transformers_utils.tokenizer import ( - AnyTokenizer, + TokenizerLike, cached_tokenizer_from_config, encode_tokens, ) @@ -284,7 +284,7 @@ class BaseNanoNemotronVLProcessor(ABC): def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *args, max_num_tiles: int | None = None, **kwargs, @@ -434,7 +434,7 @@ class NanoNemotronVLProcessor(BaseNanoNemotronVLProcessor): def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, max_num_tiles: int | None = None, min_dynamic_patch: int | None = None, @@ -645,7 +645,7 @@ def get_video_repl( tokens_per_frame: list[int], frames_indices: list[int], frame_duration_ms: int, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, img_start_token_ids: list[int], img_end_token_ids: list[int], img_context_token_ids: list[int], @@ -670,7 +670,7 @@ def get_video_repl( tokens_per_frame (list[int]): number of tokens per frame frames_indices (list[int]): frame indices frame_duration_ms (int): duration of each frame in milliseconds - tokenizer (AnyTokenizer): tokenizer to use for tokenizing frame separators + tokenizer (TokenizerLike): tokenizer to use for tokenizing frame separators img_start_token_ids (list[int]): pre-tokenized IMG_START tokens img_end_token_ids (list[int]): pre-tokenized IMG_END tokens img_context_token_ids (list[int]): pre-tokenized IMG_CONTEXT tokens diff --git a/vllm/model_executor/models/nemotron_vl.py b/vllm/model_executor/models/nemotron_vl.py index 5a1dda8aac2c..262cfb8836fa 100644 --- a/vllm/model_executor/models/nemotron_vl.py +++ b/vllm/model_executor/models/nemotron_vl.py @@ -35,7 +35,7 @@ from vllm.multimodal.processing import PromptUpdateDetails from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import cached_image_processor_from_config -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from .interfaces import ( MultiModalEmbeddings, @@ -203,7 +203,7 @@ class NemotronVLProcessor(InternVLProcessor): def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, image_processor: BaseImageProcessorFast, *, min_dynamic_patch: int | None = None, diff --git a/vllm/model_executor/models/opencua.py b/vllm/model_executor/models/opencua.py index 121bf896fa6b..ecff5572757a 100644 --- a/vllm/model_executor/models/opencua.py +++ b/vllm/model_executor/models/opencua.py @@ -31,7 +31,7 @@ PromptReplacement, PromptUpdate, ) -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from .qwen2_5_vl import ( Qwen2_5_VisionTransformer as OpenCUAVisionTransformer, @@ -79,7 +79,7 @@ def check_argument_for_proper_class(self, attribute_name: str, arg: object) -> N def __init__( self, vision_config: dict, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, **kwargs, ): image_processor = Qwen2VLImageProcessor(**vision_config) diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 672659aa6042..0440017a984e 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -91,7 +91,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import ( @@ -1533,7 +1533,7 @@ class Tarsier2Processor(Qwen2VLProcessor): def __init__( self, vision_config: dict, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, **kwargs, ): self.image_processor = Tarsier2ImageProcessor(**vision_config) diff --git a/vllm/model_executor/models/skyworkr1v.py b/vllm/model_executor/models/skyworkr1v.py index d825eb3a1c13..1f77f3ffdaa2 100644 --- a/vllm/model_executor/models/skyworkr1v.py +++ b/vllm/model_executor/models/skyworkr1v.py @@ -47,7 +47,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP @@ -282,7 +282,7 @@ class SkyworkR1VProcessor: def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, min_dynamic_patch: int | None = None, max_dynamic_patch: int | None = None, diff --git a/vllm/model_executor/models/step3_vl.py b/vllm/model_executor/models/step3_vl.py index 1c60cb414812..158a8fb21815 100644 --- a/vllm/model_executor/models/step3_vl.py +++ b/vllm/model_executor/models/step3_vl.py @@ -44,7 +44,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs import Step3VisionEncoderConfig -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP @@ -321,7 +321,7 @@ class Step3VLProcessor: def __init__( self, config: PretrainedConfig, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> None: super().__init__() diff --git a/vllm/multimodal/processing.py b/vllm/multimodal/processing.py index 691eff9acf86..630c2cfb7e98 100644 --- a/vllm/multimodal/processing.py +++ b/vllm/multimodal/processing.py @@ -24,7 +24,11 @@ from vllm.logger import init_logger from vllm.transformers_utils.processor import cached_processor_from_config -from vllm.transformers_utils.tokenizer import AnyTokenizer, decode_tokens, encode_tokens +from vllm.transformers_utils.tokenizer import ( + TokenizerLike, + decode_tokens, + encode_tokens, +) from vllm.utils.collection_utils import flatten_2d_lists, full_groupby from vllm.utils.func_utils import get_allowed_kwarg_only_overrides from vllm.utils.jsontree import JSONTree, json_map_leaves @@ -76,7 +80,7 @@ @lru_cache(maxsize=2048) def _cached_encode( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, text: str, *, add_special_tokens: bool | None = None, @@ -86,7 +90,7 @@ def _cached_encode( @lru_cache(maxsize=2048) def _cached_decode( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, token_ids: tuple[int, ...], *, skip_special_tokens: bool | None = None, @@ -96,14 +100,14 @@ def _cached_decode( ) -def _seq2text(tokenizer: AnyTokenizer, seq: PromptSeq) -> str: +def _seq2text(tokenizer: TokenizerLike, seq: PromptSeq) -> str: if isinstance(seq, str): return seq return _cached_decode(tokenizer, tuple(seq)) -def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]: +def _seq2tokens(tokenizer: TokenizerLike, seq: PromptSeq) -> list[int]: if isinstance(seq, str): return _cached_encode(tokenizer, seq, add_special_tokens=False) @@ -113,7 +117,7 @@ def _seq2tokens(tokenizer: AnyTokenizer, seq: PromptSeq) -> list[int]: class _GetMatchIndex(Protocol): def __call__( self, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, prompt: PromptSeq, start_idx: int = 0, ) -> int | None: ... @@ -143,7 +147,7 @@ def prefix(seq: PromptSeq) -> PromptIndex: """ def get_match_index( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, prompt: PromptSeq, start_idx: int = 0, ) -> int | None: @@ -199,7 +203,7 @@ class PromptUpdateDetails(Generic[_S]): full: _S """The full content.""" - is_embed: Callable[[AnyTokenizer, PromptSeq], torch.Tensor] | None = None + is_embed: Callable[[TokenizerLike, PromptSeq], torch.Tensor] | None = None """ Given [`full`][vllm.multimodal.processing.PromptUpdateDetails.full], return a boolean mask of shape `(len(full),)` indicating which positions @@ -220,7 +224,7 @@ def select_text( seq: _S, embed_text: str, ) -> "PromptUpdateDetails[_S]": - def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor: + def is_embed(tokenizer: TokenizerLike, full: PromptSeq) -> torch.Tensor: embed_token_ids = encode_tokens(tokenizer, embed_text) token_ids = _seq2tokens(tokenizer, full) @@ -236,7 +240,7 @@ def select_token_id( seq: _S, embed_token_id: int, ) -> "PromptUpdateDetails[_S]": - def is_embed(tokenizer: AnyTokenizer, full: PromptSeq) -> torch.Tensor: + def is_embed(tokenizer: TokenizerLike, full: PromptSeq) -> torch.Tensor: token_ids = _seq2tokens(tokenizer, full) return torch.tensor(token_ids) == embed_token_id @@ -522,7 +526,7 @@ class ResolvedPromptUpdate: def iter_token_matches( self, prompt: list[int], - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, start_idx: int = 0, ) -> Generator[PromptTargetMatch]: @@ -544,7 +548,7 @@ def iter_token_matches( def iter_text_matches( self, prompt: str, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, start_idx: int = 0, ) -> Generator[PromptTargetMatch]: @@ -566,7 +570,7 @@ def iter_text_matches( def iter_matches( self, prompt: list[int] | str, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, start_idx: int = 0, ) -> Generator[PromptTargetMatch]: @@ -675,7 +679,7 @@ def to_range(self) -> PlaceholderRange: def _find_matches( prompt: _S, mm_prompt_updates: "MultiModalPromptUpdates", - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, *, prev_end_idx: int = 0, current_result: "MultiModalPromptUpdatesApplyResult", @@ -740,7 +744,7 @@ def _all_items_found( def _apply_matches( prompt: _S, mm_prompt_updates: "MultiModalPromptUpdates", - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> tuple[list[_S], "MultiModalPromptUpdatesApplyResult"]: prompt_len = len(prompt) mm_item_counts = {m: len(items) for m, items in mm_prompt_updates.items()} @@ -810,7 +814,7 @@ def _apply_matches( def apply_token_matches( prompt: list[int], mm_prompt_updates: "MultiModalPromptUpdates", - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> tuple[list[int], "MultiModalPromptUpdatesApplyResult"]: """ Apply the updates in `mm_prompt_updates` to `prompt`. @@ -827,7 +831,7 @@ def apply_token_matches( def apply_text_matches( prompt: str, mm_prompt_updates: "MultiModalPromptUpdates", - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> tuple[str, "MultiModalPromptUpdatesApplyResult"]: """ Apply the updates in `mm_prompt_updates` to `prompt`. @@ -844,7 +848,7 @@ def apply_text_matches( def _iter_placeholders( prompt: list[int], mm_prompt_updates: "MultiModalPromptUpdates", - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> Iterable[PlaceholderFeaturesInfo]: """ Yield each set of placeholder tokens found in `prompt`. @@ -913,7 +917,7 @@ def _iter_placeholders( def find_mm_placeholders( prompt: list[int], mm_prompt_updates: "MultiModalPromptUpdates", - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, ) -> Mapping[str, list[PlaceholderFeaturesInfo]]: it = _iter_placeholders(prompt, mm_prompt_updates, tokenizer) return dict(full_groupby_modality(it)) @@ -934,7 +938,7 @@ class InputProcessingContext: model_config: ModelConfig """The configuration of the model.""" - tokenizer: AnyTokenizer + tokenizer: TokenizerLike """The tokenizer used to tokenize the inputs.""" @overload @@ -1150,7 +1154,7 @@ def __init__(self, ctx: InputProcessingContext) -> None: def model_id(self) -> str: return self.ctx.model_config.model - def get_tokenizer(self) -> AnyTokenizer: + def get_tokenizer(self) -> TokenizerLike: return self.ctx.tokenizer def get_hf_config(self) -> PretrainedConfig: diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py index 8f9276e84640..e557e24f551a 100644 --- a/vllm/multimodal/registry.py +++ b/vllm/multimodal/registry.py @@ -2,13 +2,16 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from collections.abc import Mapping from dataclasses import dataclass -from typing import TYPE_CHECKING, Generic, Protocol, TypeVar +from typing import TYPE_CHECKING, Generic, Protocol, TypeVar, cast import torch.nn as nn from vllm.config.multimodal import BaseDummyOptions from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import AnyTokenizer, cached_tokenizer_from_config +from vllm.transformers_utils.tokenizer import ( + TokenizerLike, + cached_tokenizer_from_config, +) from vllm.utils.collection_utils import ClassRegistry from .cache import BaseMultiModalProcessorCache @@ -235,17 +238,20 @@ def _get_model_cls(self, model_config: "ModelConfig"): def _create_processing_ctx( self, model_config: "ModelConfig", - tokenizer: AnyTokenizer | None = None, + tokenizer: TokenizerLike | None = None, ) -> InputProcessingContext: - if tokenizer is None and not model_config.skip_tokenizer_init: + if model_config.skip_tokenizer_init: + tokenizer = cast(TokenizerLike, object()) + elif tokenizer is None: tokenizer = cached_tokenizer_from_config(model_config) + return InputProcessingContext(model_config, tokenizer) def _create_processing_info( self, model_config: "ModelConfig", *, - tokenizer: AnyTokenizer | None = None, + tokenizer: TokenizerLike | None = None, ) -> BaseProcessingInfo: model_cls = self._get_model_cls(model_config) factories = self._processor_factories[model_cls] @@ -256,7 +262,7 @@ def create_processor( self, model_config: "ModelConfig", *, - tokenizer: AnyTokenizer | None = None, + tokenizer: TokenizerLike | None = None, cache: BaseMultiModalProcessorCache | None = None, ) -> BaseMultiModalProcessor[BaseProcessingInfo]: """ diff --git a/vllm/reasoning/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py index d26e4ffc9c16..f15c585b4aec 100644 --- a/vllm/reasoning/abs_reasoning_parsers.py +++ b/vllm/reasoning/abs_reasoning_parsers.py @@ -19,12 +19,12 @@ DeltaMessage, ResponsesRequest, ) - from vllm.transformers_utils.tokenizer import AnyTokenizer + from vllm.transformers_utils.tokenizer import TokenizerLike else: ChatCompletionRequest = Any DeltaMessage = Any ResponsesRequest = Any - AnyTokenizer = Any + TokenizerLike = Any logger = init_logger(__name__) @@ -37,7 +37,7 @@ class ReasoningParser: It is used to extract reasoning content from the model output. """ - def __init__(self, tokenizer: AnyTokenizer, *args, **kwargs): + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): self.model_tokenizer = tokenizer @cached_property diff --git a/vllm/reasoning/basic_parsers.py b/vllm/reasoning/basic_parsers.py index 026894773272..2af4eaf5f051 100644 --- a/vllm/reasoning/basic_parsers.py +++ b/vllm/reasoning/basic_parsers.py @@ -7,7 +7,7 @@ from vllm.entrypoints.openai.protocol import DeltaMessage from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike if TYPE_CHECKING: from vllm.entrypoints.openai.protocol import ( @@ -43,7 +43,7 @@ def end_token(self) -> str: """The token that ends reasoning content.""" raise NotImplementedError - def __init__(self, tokenizer: AnyTokenizer, *args, **kwargs): + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): super().__init__(tokenizer, *args, **kwargs) if not self.model_tokenizer: diff --git a/vllm/reasoning/minimax_m2_reasoning_parser.py b/vllm/reasoning/minimax_m2_reasoning_parser.py index 30f5f2f88caf..f40f4570952a 100644 --- a/vllm/reasoning/minimax_m2_reasoning_parser.py +++ b/vllm/reasoning/minimax_m2_reasoning_parser.py @@ -11,7 +11,7 @@ from vllm.logger import init_logger from vllm.reasoning.abs_reasoning_parsers import ReasoningParser from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike logger = init_logger(__name__) @@ -37,7 +37,7 @@ class MiniMaxM2AppendThinkReasoningParser(ReasoningParser): Reasoning parser for MiniMax M2 model. """ - def __init__(self, tokenizer: AnyTokenizer, *args, **kwargs): + def __init__(self, tokenizer: TokenizerLike, *args, **kwargs): super().__init__(tokenizer, *args, **kwargs) self.end_token_id = self.vocab.get("") diff --git a/vllm/reasoning/olmo3_reasoning_parser.py b/vllm/reasoning/olmo3_reasoning_parser.py index 7149f8c4123b..2e9e91c148a5 100644 --- a/vllm/reasoning/olmo3_reasoning_parser.py +++ b/vllm/reasoning/olmo3_reasoning_parser.py @@ -9,7 +9,7 @@ import regex as re if TYPE_CHECKING: - from vllm.transformers_utils.tokenizer import AnyTokenizer + from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, @@ -220,7 +220,7 @@ class Olmo3ReasoningParser(ReasoningParser): token is missing from generation. """ - def __init__(self, tokenizer: "AnyTokenizer", *args, **kwargs): + def __init__(self, tokenizer: "TokenizerLike", *args, **kwargs): super().__init__(tokenizer, *args, **kwargs) self.think_start = r"" diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 8de961e62db1..9f793ada4f30 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -13,7 +13,7 @@ from vllm.logger import init_logger from vllm.logits_process import LogitsProcessor -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.v1.serial_utils import PydanticMsgspecMixin logger = init_logger(__name__) @@ -477,7 +477,7 @@ def update_from_generation_config( eos_ids.update(self.stop_token_ids) self.stop_token_ids = list(eos_ids) - def update_from_tokenizer(self, tokenizer: AnyTokenizer) -> None: + def update_from_tokenizer(self, tokenizer: TokenizerLike) -> None: if not self.bad_words: return self._bad_words_token_ids = [] diff --git a/vllm/transformers_utils/detokenizer_utils.py b/vllm/transformers_utils/detokenizer_utils.py index 560526bfd823..33d7fbacd66c 100644 --- a/vllm/transformers_utils/detokenizer_utils.py +++ b/vllm/transformers_utils/detokenizer_utils.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from .tokenizer import AnyTokenizer +from .tokenizer import TokenizerLike def _replace_none_with_empty(tokens: list[str | None]): @@ -12,7 +12,7 @@ def _replace_none_with_empty(tokens: list[str | None]): def _convert_tokens_to_string_with_added_encoders( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, output_tokens: list[str], skip_special_tokens: bool, spaces_between_special_tokens: bool, @@ -57,7 +57,7 @@ def _convert_tokens_to_string_with_added_encoders( def convert_prompt_ids_to_tokens( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, prompt_ids: list[int], skip_special_tokens: bool = False, ) -> tuple[list[str], int, int]: @@ -81,7 +81,7 @@ def convert_prompt_ids_to_tokens( def convert_ids_list_to_tokens( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, token_ids: list[int], ) -> list[str]: """Detokenize the input ids individually. @@ -108,7 +108,7 @@ def convert_ids_list_to_tokens( # https://github.com/huggingface/text-generation-inference/blob/v0.9.4/server/text_generation_server/models/model.py#L62C9-L62C15 # under Apache 2.0 license def detokenize_incrementally( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, all_input_ids: list[int], prev_tokens: list[str] | None, prefix_offset: int, diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 9eb7fe37912b..a7c16ab60da3 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -5,12 +5,13 @@ import copy import importlib.util import os +import warnings from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Any, TypeAlias +from typing import TYPE_CHECKING, Any import huggingface_hub -from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast +from transformers import AutoTokenizer, PreTrainedTokenizerBase from typing_extensions import assert_never from vllm import envs @@ -18,6 +19,7 @@ from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf from vllm.transformers_utils.repo_utils import list_filtered_repo_files +from vllm.transformers_utils.tokenizer_base import TokenizerLike from vllm.transformers_utils.tokenizers import MistralTokenizer from vllm.transformers_utils.utils import ( check_gguf_file, @@ -28,18 +30,31 @@ if TYPE_CHECKING: from vllm.config import ModelConfig - from vllm.transformers_utils.tokenizer_base import TokenizerBase else: ModelConfig = Any - TokenizerBase = Any logger = init_logger(__name__) -AnyTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast | TokenizerBase + +def __getattr__(name: str): + # TODO: Move AnyTokenizer into this file + # and move TokenizerRegistry into `registry.py` with a deprecation + if name == "AnyTokenizer": + warnings.warn( + "`vllm.transformers_utils.tokenizer.AnyTokenizer` has been renamed to " + "`vllm.transformers_utils.tokenizer.TokenizerLike`. " + "The old name will be removed in v0.13.", + DeprecationWarning, + stacklevel=2, + ) + + return TokenizerLike + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") def decode_tokens( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, token_ids: list[int], *, skip_special_tokens: bool | None = None, @@ -58,7 +73,7 @@ def decode_tokens( def encode_tokens( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, text: str, *, truncation: bool | None = None, @@ -86,7 +101,7 @@ def encode_tokens( return tokenizer.encode(text, **kw_args) -def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer: +def get_cached_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike: """ By default, transformers will recompute multiple tokenizer properties each time they are called, leading to a significant slowdown. @@ -96,7 +111,6 @@ def get_cached_tokenizer(tokenizer: AnyTokenizer) -> AnyTokenizer: tokenizer_all_special_ids = tokenizer.all_special_ids tokenizer_all_special_tokens = tokenizer.all_special_tokens - tokenizer_all_special_tokens_extended = tokenizer.all_special_tokens_extended tokenizer_vocab = tokenizer.get_vocab() tokenizer_len = len(tokenizer) @@ -118,10 +132,6 @@ def all_special_ids(self) -> list[int]: def all_special_tokens(self) -> list[str]: return tokenizer_all_special_tokens - @property - def all_special_tokens_extended(self) -> list[str]: - return tokenizer_all_special_tokens_extended - @property def max_token_id(self) -> int: return max_token_id @@ -149,7 +159,7 @@ def get_tokenizer( revision: str | None = None, download_dir: str | None = None, **kwargs, -) -> AnyTokenizer: +) -> TokenizerLike: """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" if envs.VLLM_USE_MODELSCOPE: # download model from ModelScope hub, @@ -211,7 +221,7 @@ def get_tokenizer( if len(files_list) > 0: tokenizer_mode = "mistral" - tokenizer: AnyTokenizer + tokenizer: TokenizerLike if tokenizer_mode == "mistral": logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}") tokenizer = MistralTokenizer.from_pretrained( @@ -265,12 +275,13 @@ def get_tokenizer( if isinstance(encoder_config, dict) and encoder_config.get( "do_lower_case", False ): + assert isinstance(tokenizer, PreTrainedTokenizerBase) special_tokens_map = { k: v.lower() for k, v in tokenizer.special_tokens_map.items() } tokenizer.add_special_tokens(special_tokens_map) - if not isinstance(tokenizer, PreTrainedTokenizerFast): + if not tokenizer.is_fast: logger.warning( "Using a slow tokenizer. This might cause a significant " "slowdown. Consider using a fast tokenizer instead." diff --git a/vllm/transformers_utils/tokenizer_base.py b/vllm/transformers_utils/tokenizer_base.py index 7421eb534808..cb7b76ee6155 100644 --- a/vllm/transformers_utils/tokenizer_base.py +++ b/vllm/transformers_utils/tokenizer_base.py @@ -2,73 +2,64 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib -from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Protocol + +from typing_extensions import Self, runtime_checkable if TYPE_CHECKING: from vllm.entrypoints.chat_utils import ChatCompletionMessageParam -class TokenizerBase(ABC): - @property - @abstractmethod - def all_special_tokens_extended(self) -> list[str]: - raise NotImplementedError() +@runtime_checkable +class TokenizerLike(Protocol): + @classmethod + def from_pretrained( + cls, + pretrained_model_name_or_path: str, + /, + *, + revision: str | None = None, + ) -> "Self": + raise NotImplementedError @property - @abstractmethod def all_special_tokens(self) -> list[str]: - raise NotImplementedError() + raise NotImplementedError @property - @abstractmethod def all_special_ids(self) -> list[int]: - raise NotImplementedError() + raise NotImplementedError @property - @abstractmethod def bos_token_id(self) -> int: - raise NotImplementedError() + raise NotImplementedError @property - @abstractmethod def eos_token_id(self) -> int: - raise NotImplementedError() - - @property - @abstractmethod - def sep_token(self) -> str: - raise NotImplementedError() + raise NotImplementedError @property - @abstractmethod - def pad_token(self) -> str: - raise NotImplementedError() - - @property - @abstractmethod def is_fast(self) -> bool: - raise NotImplementedError() + raise NotImplementedError @property - @abstractmethod def vocab_size(self) -> int: - raise NotImplementedError() + raise NotImplementedError @property - @abstractmethod def max_token_id(self) -> int: - raise NotImplementedError() + raise NotImplementedError @property - @abstractmethod def truncation_side(self) -> str: - raise NotImplementedError() + raise NotImplementedError + + def __hash__(self) -> int: + return hash(id(self)) def __len__(self) -> int: return self.vocab_size - @abstractmethod def __call__( self, text: str | list[str] | list[int], @@ -77,26 +68,14 @@ def __call__( truncation: bool = False, max_length: int | None = None, ): - raise NotImplementedError() + raise NotImplementedError - @abstractmethod def get_vocab(self) -> dict[str, int]: - raise NotImplementedError() + raise NotImplementedError - @abstractmethod def get_added_vocab(self) -> dict[str, int]: - raise NotImplementedError() - - @abstractmethod - def encode_one( - self, - text: str, - truncation: bool = False, - max_length: int | None = None, - ) -> list[int]: - raise NotImplementedError() + raise NotImplementedError - @abstractmethod def encode( self, text: str, @@ -104,32 +83,28 @@ def encode( max_length: int | None = None, add_special_tokens: bool | None = None, ) -> list[int]: - raise NotImplementedError() + raise NotImplementedError - @abstractmethod def apply_chat_template( self, messages: list["ChatCompletionMessageParam"], tools: list[dict[str, Any]] | None = None, **kwargs, ) -> list[int]: - raise NotImplementedError() + raise NotImplementedError - @abstractmethod def convert_tokens_to_string(self, tokens: list[str]) -> str: - raise NotImplementedError() + raise NotImplementedError - @abstractmethod def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str: - raise NotImplementedError() + raise NotImplementedError - @abstractmethod def convert_ids_to_tokens( self, ids: list[int], skip_special_tokens: bool = True, ) -> list[str]: - raise NotImplementedError() + raise NotImplementedError class TokenizerRegistry: @@ -145,7 +120,7 @@ def get_tokenizer( tokenizer_name: str, *args, **kwargs, - ) -> TokenizerBase: + ) -> TokenizerLike: tokenizer_cls = TokenizerRegistry.REGISTRY.get(tokenizer_name) if tokenizer_cls is None: raise ValueError(f"Tokenizer {tokenizer_name} not found.") diff --git a/vllm/transformers_utils/tokenizers/mistral.py b/vllm/transformers_utils/tokenizers/mistral.py index caff43c55ce8..bfa16310106a 100644 --- a/vllm/transformers_utils/tokenizers/mistral.py +++ b/vllm/transformers_utils/tokenizers/mistral.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Any, cast from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer_base import TokenizerBase +from vllm.transformers_utils.tokenizer_base import TokenizerLike if TYPE_CHECKING: from mistral_common.protocol.instruct.request import ( @@ -163,7 +163,7 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int: return tokenizer.unk_id -class MistralTokenizer(TokenizerBase): +class MistralTokenizer(TokenizerLike): def __init__(self, tokenizer: "TransformersMistralTokenizer") -> None: from mistral_common.protocol.instruct.validator import ValidationMode from mistral_common.tokens.tokenizers.sentencepiece import ( @@ -254,10 +254,6 @@ def _get_special_tokens(self, all_special_ids: list[int]) -> list[str]: # the following attributes are set to fit vLLM's design and are used # by the structured output backends. - @property - def all_special_tokens_extended(self) -> list[str]: - return self.all_special_tokens - @property def all_special_tokens(self) -> list[str]: return self._special_tokens @@ -274,14 +270,6 @@ def bos_token_id(self) -> int: def eos_token_id(self) -> int: return self.tokenizer.eos_id - @property - def sep_token(self) -> str: - raise NotImplementedError() - - @property - def pad_token(self) -> str: - return self.transformers_tokenizer.pad_token - @property def is_fast(self) -> bool: return True @@ -296,11 +284,14 @@ def max_token_id(self) -> int: @property def truncation_side(self) -> str: - raise NotImplementedError() + return self.transformers_tokenizer.truncation_side def _is_special_token_id(self, token_id: int) -> bool: return token_id in self._special_token_ids_set + def __hash__(self) -> int: + return hash(id(self)) + def __len__(self) -> int: return self.vocab_size @@ -345,17 +336,6 @@ def get_added_vocab(self) -> dict[str, int]: # Mistral tokenizers have no added vocabulary return {} - def encode_one( - self, - text: str, - truncation: bool = False, - max_length: int | None = None, - ) -> list[int]: - # Mistral Tokenizers should not add special tokens - return self.transformers_tokenizer.encode( - text, add_special_tokens=False, truncation=truncation, max_length=max_length - ) - def encode( self, text: str, diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 827a2736af28..9ebd2097e253 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -27,7 +27,7 @@ from vllm.tasks import SupportedTask from vllm.tracing import init_tracer from vllm.transformers_utils.config import maybe_register_config_serialize_by_value -from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs +from vllm.transformers_utils.tokenizer import TokenizerLike, init_tokenizer_from_configs from vllm.usage.usage_lib import UsageContext from vllm.utils.async_utils import cancel_task_threadsafe from vllm.utils.collection_utils import as_list @@ -119,9 +119,10 @@ def __init__( ) # OutputProcessor (converts EngineCoreOutputs --> RequestOutput). - stream_interval = self.vllm_config.scheduler_config.stream_interval self.output_processor = OutputProcessor( - self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval + self.tokenizer, + log_stats=self.log_stats, + stream_interval=self.vllm_config.scheduler_config.stream_interval, ) endpoint = self.observability_config.otlp_traces_endpoint if endpoint is not None: @@ -698,14 +699,14 @@ async def encode( raise EngineGenerateError() from e @property - def tokenizer(self) -> AnyTokenizer | None: + def tokenizer(self) -> TokenizerLike | None: return self.processor.tokenizer @tokenizer.setter - def tokenizer(self, tokenizer: AnyTokenizer | None) -> None: + def tokenizer(self, tokenizer: TokenizerLike | None) -> None: self.processor.tokenizer = tokenizer - async def get_tokenizer(self) -> AnyTokenizer: + async def get_tokenizer(self) -> TokenizerLike: if self.tokenizer is None: raise ValueError( "Unable to get tokenizer because skip_tokenizer_init is True" diff --git a/vllm/v1/engine/detokenizer.py b/vllm/v1/engine/detokenizer.py index b7a24096bf15..c55240c40f6f 100644 --- a/vllm/v1/engine/detokenizer.py +++ b/vllm/v1/engine/detokenizer.py @@ -10,7 +10,7 @@ from vllm.logger import init_logger from vllm.transformers_utils.detokenizer_utils import ( - AnyTokenizer, + TokenizerLike, convert_prompt_ids_to_tokens, detokenize_incrementally, ) @@ -45,7 +45,7 @@ def get_next_output_text(self, finished: bool, delta: bool) -> str: @classmethod def from_new_request( cls, - tokenizer: AnyTokenizer | None, + tokenizer: TokenizerLike | None, request: EngineCoreRequest, ) -> "IncrementalDetokenizer": assert request.sampling_params is not None @@ -256,7 +256,7 @@ def _protected_step(self, next_token_id: int) -> str | None: class SlowIncrementalDetokenizer(BaseIncrementalDetokenizer): - def __init__(self, tokenizer: AnyTokenizer, request: EngineCoreRequest): + def __init__(self, tokenizer: TokenizerLike, request: EngineCoreRequest): super().__init__(request) self.tokenizer = tokenizer diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py new file mode 100644 index 000000000000..6fac7f5118ea --- /dev/null +++ b/vllm/v1/engine/input_processor.py @@ -0,0 +1,637 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import time +from collections.abc import Mapping +from typing import Any, Literal, cast + +from vllm.config import VllmConfig +from vllm.inputs import ProcessorInputs, PromptType, SingletonInputs +from vllm.inputs.parse import split_enc_dec_inputs +from vllm.inputs.preprocess import InputPreprocessor +from vllm.logger import init_logger +from vllm.lora.request import LoRARequest +from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry +from vllm.multimodal.cache import processor_cache_from_config +from vllm.multimodal.inputs import MultiModalFeatureSpec, MultiModalUUIDDict +from vllm.multimodal.parse import MultiModalDataParser +from vllm.multimodal.processing import EncDecMultiModalProcessor +from vllm.multimodal.utils import argsort_mm_positions +from vllm.pooling_params import PoolingParams +from vllm.sampling_params import SamplingParams +from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer +from vllm.utils import length_from_prompt_token_ids_or_embeds +from vllm.v1.engine import EngineCoreRequest +from vllm.v1.metrics.stats import MultiModalCacheStats +from vllm.v1.structured_output.backend_guidance import validate_guidance_grammar +from vllm.v1.structured_output.backend_lm_format_enforcer import ( + validate_structured_output_request_lm_format_enforcer, +) +from vllm.v1.structured_output.backend_outlines import ( + validate_structured_output_request_outlines, +) +from vllm.v1.structured_output.backend_xgrammar import validate_xgrammar_grammar + +logger = init_logger(__name__) + + +class InputProcessor: + def __init__( + self, + vllm_config: VllmConfig, + tokenizer: TokenizerLike | None, + mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, + ) -> None: + self.vllm_config = vllm_config + self.model_config = vllm_config.model_config + self.cache_config = vllm_config.cache_config + self.lora_config = vllm_config.lora_config + self.structured_outputs_config = vllm_config.structured_outputs_config + + self.generation_config_fields = self.model_config.try_get_generation_config() + + self.mm_registry = mm_registry + self.mm_processor_cache = processor_cache_from_config(vllm_config, mm_registry) + + self.input_preprocessor = InputPreprocessor( + self.model_config, + tokenizer, + mm_registry, + mm_processor_cache=self.mm_processor_cache, + ) + + @property + def tokenizer(self) -> TokenizerLike | None: + return self.input_preprocessor.tokenizer + + @tokenizer.setter + def tokenizer(self, tokenizer: TokenizerLike | None) -> None: + self.input_preprocessor.tokenizer = tokenizer + + def _validate_logprobs( + self, + params: SamplingParams, + ) -> None: + max_logprobs = self.model_config.max_logprobs + if max_logprobs == -1: + max_logprobs = self.model_config.get_vocab_size() + + # Validate sample logprobs. + if params.logprobs: + num_logprobs = params.logprobs + if num_logprobs == -1: + num_logprobs = self.model_config.get_vocab_size() + if num_logprobs > max_logprobs: + raise ValueError( + f"Requested sample logprobs of {num_logprobs}, " + f"which is greater than max allowed: {max_logprobs}" + ) + + # Validate prompt logprobs. + if params.prompt_logprobs: + num_prompt_logprobs = params.prompt_logprobs + if num_prompt_logprobs == -1: + num_prompt_logprobs = self.model_config.get_vocab_size() + if num_prompt_logprobs > max_logprobs: + raise ValueError( + f"Requested prompt logprobs of {num_prompt_logprobs}, " + f"which is greater than max allowed: {max_logprobs}" + ) + + def _validate_sampling_params( + self, + params: SamplingParams, + ) -> None: + self._validate_structured_output(params) + self._validate_logit_bias(params) + + if params.allowed_token_ids is None: + return + if not params.allowed_token_ids: + raise ValueError("allowed_token_ids is not None and empty!") + if self.tokenizer is None: + # When skip_tokenizer_init=True, we can't validate token IDs + # Skip validation and let the model handle invalid tokens + return + vocab_size = len(self.tokenizer) + if not all(0 <= tid < vocab_size for tid in params.allowed_token_ids): + raise ValueError("allowed_token_ids contains out-of-vocab token id!") + + def _validate_logit_bias( + self, + params: SamplingParams, + ) -> None: + """Validate logit_bias token IDs are within vocabulary range.""" + if not params.logit_bias: + return + + vocab_size = self.model_config.get_vocab_size() + invalid_token_ids = [] + + for token_id in params.logit_bias: + if token_id < 0 or token_id >= vocab_size: + invalid_token_ids.append(token_id) + + if invalid_token_ids: + raise ValueError( + f"token_id(s) {invalid_token_ids} in logit_bias contain " + f"out-of-vocab token ids. Vocabulary size: {vocab_size}" + ) + + def _validate_supported_sampling_params( + self, + params: SamplingParams, + ) -> None: + # Logits processors not supported. + if params.logits_processors: + raise ValueError( + "vLLM V1 does not support per request user provided logits processors." + ) + # Async scheduling + spec decode currently incompatible with some + # sampling parameters. + if ( + self.vllm_config.speculative_config is not None + and self.vllm_config.scheduler_config.async_scheduling + and ( + params.frequency_penalty != 0.0 + or params.presence_penalty != 0.0 + or params.repetition_penalty != 1.0 + or params.bad_words_token_ids + or params.structured_outputs + ) + ): + raise ValueError( + "async scheduling with spec decoding doesn't yet support " + "penalties, bad words or structured outputs in sampling parameters." + ) + + def _validate_params( + self, + params: SamplingParams | PoolingParams, + ): + """ + Validate supported SamplingParam. + Should raise ValueError if unsupported for API Server. + """ + + if isinstance(params, PoolingParams): + return + + self._validate_logprobs(params) + self._validate_sampling_params(params) + self._validate_supported_sampling_params(params) + + def _validate_multi_modal_uuids(self, prompt: PromptType) -> None: + """ + Validate that user-provided multi_modal_uuids align with + multi_modal_data in the incoming request prompt(s). + Only checks lengths; `None` entries are allowed and will be + auto-hashed downstream. + """ + + def _validate_single_prompt(single_prompt: dict | str) -> None: + if not isinstance(single_prompt, dict): + return + mm_data = single_prompt.get("multi_modal_data") + mm_uuids = single_prompt.get("multi_modal_uuids") + if not mm_data or not mm_uuids: + return + + for modality, items in mm_data.items(): + if modality in mm_uuids: + data_len = len(items) if isinstance(items, list) else 1 + uuid_len = ( + len(mm_uuids[modality]) + if isinstance(mm_uuids[modality], list) + else 1 + ) + if uuid_len != data_len: + raise ValueError( + f"multi_modal_uuids for modality '{modality}' " + "must have same length as data: got " + f"{uuid_len} uuids vs " + f"{data_len} items." + ) + else: + raise ValueError( + f"multi_modal_uuids for modality '{modality}' must " + "be provided if multi_modal_data is provided." + ) + + # Handle explicit encoder/decoder prompts or singleton prompt + if isinstance(prompt, dict) and "encoder_prompt" in prompt: + enc = prompt.get("encoder_prompt") + dec = prompt.get("decoder_prompt") + if enc is not None: + _validate_single_prompt(cast(dict | str, enc)) + if dec is not None: + _validate_single_prompt(cast(dict | str, dec)) + else: + _validate_single_prompt(prompt) # type: ignore[arg-type] + + def _validate_lora(self, lora_request: LoRARequest | None) -> None: + if lora_request is None: + return + + # LoRA request passed in while LoRA is not enabled + if not self.lora_config: + raise ValueError( + f"Got lora_request {lora_request} but LoRA is not enabled!" + ) + + if self.tokenizer is not None: + logger.warning_once( + "vLLM has deprecated support for supporting different " + "tokenizers for different LoRAs. By default, vLLM uses base " + "model's tokenizer. If you are using a LoRA " + "with its own tokenizer, consider specifying `--tokenizer " + "[lora_path]` to use the LoRA tokenizer." + ) + + def _validate_structured_output(self, params: SamplingParams) -> None: + if not params.structured_outputs or not self.structured_outputs_config: + return + + if self.model_config.skip_tokenizer_init and params.structured_outputs: + raise ValueError( + "Structured outputs requires a tokenizer so it can't be used with 'skip_tokenizer_init'" # noqa: E501 + ) + + backend = self.structured_outputs_config.backend + if _backend := params.structured_outputs._backend: + # Request-level backend selection is not supported. + # The values may differ if `params` is reused and was set + # to a specific backend based on `auto` behavior in a previous + # request. We remember that it was set as a result of `auto` + # using the `_backend_was_auto` field set in the params. + if backend != _backend and not ( + backend == "auto" and params.structured_outputs._backend_was_auto + ): + raise ValueError( + "Request-level structured output backend selection is not " + f"supported. The request specified '{_backend}', but vLLM " + f"was initialised with '{backend}'. This error can be " + "resolved by removing '_backend' from the request." + ) + else: + params.structured_outputs._backend = backend + + # Request content validation + if ( + isinstance(params.structured_outputs.choice, list) + and not params.structured_outputs.choice + ): + # It is invalid for choice to be an empty list + raise ValueError( + f"Choice '{params.structured_outputs.choice}' cannot be an empty list" # noqa: E501 + ) + # Reject empty string grammar early to avoid engine-side crashes + if ( + isinstance(params.structured_outputs.grammar, str) + and params.structured_outputs.grammar.strip() == "" + ): + raise ValueError("structured_outputs.grammar cannot be an empty string") + + if backend.startswith("xgrammar"): + # xgrammar with no fallback + validate_xgrammar_grammar(params) + elif backend.startswith("guidance"): + # TODO: ideally we would have the LLTokenizer here as Lark syntax + # allows <|special_token|> and similar, see + # https://github.com/guidance-ai/llguidance/blob/main/docs/syntax.md#special-tokens + # Without tokenizer these are disallowed in grammars. + if isinstance(self.tokenizer, MistralTokenizer): + raise ValueError( + "Mistral tokenizer is not supported for the 'guidance' " + "structured output backend. Please use ['xgrammar', 'outlines'] " + "backends or tokenizer_mode='hf' instead." + ) + validate_guidance_grammar(params, tokenizer=None) + elif backend == "outlines": + # outlines backend + validate_structured_output_request_outlines(params) + elif backend == "lm-format-enforcer": + # lm format enforcer backend + if isinstance(self.tokenizer, MistralTokenizer): + raise ValueError( + "Mistral tokenizer is not supported for the 'lm-format-enforcer' " + "structured output backend. Please use ['xgrammar', 'outlines'] " + "backends or tokenizer_mode='hf' instead." + ) + validate_structured_output_request_lm_format_enforcer(params) + else: + # NOTE: backend must be "auto" here, because we have + # checked supported_backends above. + # In this mode, we set opinionated defaults based on what we think + # will satisfy the most use cases without having to worry about + # this setting. We include fallback behavior here, but not with any + # other setting where a specific backend was specified. + try: + validate_xgrammar_grammar(params) + params.structured_outputs._backend = "xgrammar" + except ValueError: + # The request either failed validation + # or includes some jsonschema feature(s) that + # are not supported in xgrammar. + if isinstance(self.tokenizer, MistralTokenizer): + # Fall back to outlines if the tokenizer is Mistral + validate_structured_output_request_outlines(params) + params.structured_outputs._backend = "outlines" + else: + # Fall back to guidance by default. + validate_guidance_grammar(params, tokenizer=None) + params.structured_outputs._backend = "guidance" + # Remember that this backend was set automatically + params.structured_outputs._backend_was_auto = True + + def _maybe_build_mm_uuids( + self, + request_id: str, + prompt: PromptType, + ) -> MultiModalUUIDDict | None: + """Build per-item multimodal hash overrides when enabled. In this case, + multimodal data items are identified by their request id, modality and + index rather than their content. + + Returns a dictionary of modality -> list[str] of overrides, or None if + disabled or no multimodal data is present. + """ + + def _extract_mm_data(p: PromptType): + if isinstance(p, dict) and "encoder_prompt" in p: + enc = p.get("encoder_prompt") + if isinstance(enc, dict): + return enc.get("multi_modal_data") + return None + if isinstance(p, dict): + return p.get("multi_modal_data") + return None + + mm_data = _extract_mm_data(prompt) + if not mm_data: + return None + + mm_uuids: dict[str, list[str | None] | str] = {} + for modality, data in mm_data.items(): + # Hash each item for embedding inputs. + n = ( + len(data) + if isinstance(data, list) or MultiModalDataParser.is_embeddings(data) + else 1 + ) + mm_uuids[modality] = [f"{request_id}-{modality}-{i}" for i in range(n)] + return mm_uuids + + def process_inputs( + self, + request_id: str, + prompt: PromptType, + params: SamplingParams | PoolingParams, + arrival_time: float | None = None, + lora_request: LoRARequest | None = None, + tokenization_kwargs: dict[str, Any] | None = None, + trace_headers: Mapping[str, str] | None = None, + priority: int = 0, + data_parallel_rank: int | None = None, + ) -> EngineCoreRequest: + self._validate_lora(lora_request) + self._validate_params(params) + + data_parallel_size = self.vllm_config.parallel_config.data_parallel_size + if data_parallel_rank is not None and not ( + 0 <= data_parallel_rank < data_parallel_size + ): + raise ValueError( + f"data_parallel_rank {data_parallel_rank} " + f"is out of range [0, {data_parallel_size})." + ) + + if arrival_time is None: + arrival_time = time.time() + + # Optionally generate multimodal hash overrides to avoid hashing + # multimodal data items by their content as their identifiers. + + # NOTE: when users explicitly turn off BOTH prefix caching and input + # processing caching, no multimodal features or embeddings will be + # reused across requests, therefore identifying multimodal data items + # by their content is no longer necessary, and we create uuids with + # request id-modality-index as multimodal hash overrides. + if ( + self.model_config.multimodal_config + and self.model_config.multimodal_config.mm_processor_cache_gb == 0 + and not self.cache_config.enable_prefix_caching + ): + mm_uuids = self._maybe_build_mm_uuids(request_id, prompt) + else: + # Otherwise, use user-provided uuids as multimodal hash overrides + # if provided. + self._validate_multi_modal_uuids(prompt) + if isinstance(prompt, dict): + mm_uuids = cast( + MultiModalUUIDDict | None, prompt.get("multi_modal_uuids") + ) + else: + mm_uuids = None + + # Process inputs, which includes: + # 1. Tokenize text prompt, with LoRA request if one exists. + # 2. For multimodal models with a merged preprocessor, preprocess + # multimodal data and expand prompt token ids accordingly. + processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess( + prompt, + tokenization_kwargs=tokenization_kwargs, + mm_uuids=mm_uuids, + ) + from vllm.platforms import current_platform + + current_platform.validate_request( + prompt=prompt, + params=params, + processed_inputs=processed_inputs, + ) + + eos_token_id = self.input_preprocessor.get_eos_token_id() + + encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs) + self._validate_model_inputs(encoder_inputs, decoder_inputs) + + # Mypy can be conservative for TypedDict unions; normalize access. + if decoder_inputs["type"] == "embeds": + prompt_token_ids = None + prompt_embeds = decoder_inputs["prompt_embeds"] + else: + prompt_token_ids = decoder_inputs["prompt_token_ids"] + prompt_embeds = None + + sampling_params = None + pooling_params = None + if isinstance(params, SamplingParams): + # TODO: can we avoid cloning here in multiproc case? + sampling_params = params.clone() + # If unset max tokens, then generate up to the max_model_len. + if sampling_params.max_tokens is None: + seq_len = length_from_prompt_token_ids_or_embeds( + prompt_token_ids, prompt_embeds + ) + sampling_params.max_tokens = self.model_config.max_model_len - seq_len + sampling_params.update_from_generation_config( + self.generation_config_fields, eos_token_id + ) + if self.tokenizer is not None: + sampling_params.update_from_tokenizer(self.tokenizer) + else: + pooling_params = params.clone() + + # Multimodal related. + mm_features: list[MultiModalFeatureSpec] | None = None + + if decoder_inputs["type"] == "multimodal": + decoder_mm_inputs = decoder_inputs["mm_kwargs"] + decoder_mm_positions = decoder_inputs["mm_placeholders"] + decoder_mm_hashes = decoder_inputs["mm_hashes"] + + # Merge and flatten multimodal placeholders, hashes and inputs + # from dictionaries to lists, and sort them by each item's position + # in the input sequence. + sorted_mm_idxs = argsort_mm_positions(decoder_mm_positions) + + mm_features = [] + for modality, idx in sorted_mm_idxs: + mm_features.append( + MultiModalFeatureSpec( + data=decoder_mm_inputs[modality][idx], + modality=modality, + identifier=decoder_mm_hashes[modality][idx], + mm_position=decoder_mm_positions[modality][idx], + ) + ) + + return EngineCoreRequest( + request_id=request_id, + prompt_token_ids=prompt_token_ids, + prompt_embeds=prompt_embeds, + mm_features=mm_features, + sampling_params=sampling_params, + pooling_params=pooling_params, + eos_token_id=eos_token_id, + arrival_time=arrival_time, + lora_request=lora_request, + cache_salt=decoder_inputs.get("cache_salt"), + priority=priority, + data_parallel_rank=data_parallel_rank, + trace_headers=trace_headers, + ) + + def _validate_model_inputs( + self, encoder_inputs: SingletonInputs | None, decoder_inputs: SingletonInputs + ): + if encoder_inputs is not None: + self._validate_model_input(encoder_inputs, prompt_type="encoder") + + self._validate_model_input(decoder_inputs, prompt_type="decoder") + + def _validate_model_input( + self, + prompt_inputs: SingletonInputs, + *, + prompt_type: Literal["encoder", "decoder"], + ): + model_config = self.model_config + + prompt_ids = ( + None + if prompt_inputs["type"] == "embeds" + else prompt_inputs["prompt_token_ids"] + ) + prompt_embeds = ( + prompt_inputs["prompt_embeds"] + if prompt_inputs["type"] == "embeds" + else None + ) + prompt_len = length_from_prompt_token_ids_or_embeds(prompt_ids, prompt_embeds) + if not prompt_ids: + if prompt_type == "encoder" and model_config.is_multimodal_model: + pass # Mllama may have empty encoder inputs for text-only data + elif prompt_inputs["type"] == "embeds": + pass # Prompt embeds should not have prompt_ids. + else: + raise ValueError(f"The {prompt_type} prompt cannot be empty") + + tokenizer = self.tokenizer + if tokenizer is not None: + max_input_id = max(prompt_ids or [], default=0) + + # NOTE: tokenizer.max_token_id is the tokenizer’s vocab size while + # self.model_config.get_vocab_size() is the model’s vocab size. + # For Qwen3 models, the language model has extra tokens that do + # not exist in the tokenizer, and vice versa for multimodal + # placeholder tokens in some multimodal models. + # See https://github.com/QwenLM/Qwen3/issues/29#issuecomment-1933720399 # noqa: E501 + # and https://github.com/vllm-project/vllm/pull/22471#discussion_r2312251421 # noqa: E501 + + # Here we take the max of the two to determine if a token id is + # truly out-of-vocabulary. + if max_input_id > max( + tokenizer.max_token_id, self.model_config.get_vocab_size() - 1 + ): + raise ValueError(f"Token id {max_input_id} is out of vocabulary") + + max_prompt_len = self.model_config.max_model_len + if prompt_len > max_prompt_len: + if prompt_type == "encoder" and model_config.is_multimodal_model: + mm_registry = self.input_preprocessor.mm_registry + mm_processor = mm_registry.create_processor( + model_config, + tokenizer=tokenizer, + ) + assert isinstance(mm_processor, EncDecMultiModalProcessor) + + if mm_processor.pad_dummy_encoder_prompt: + return # Skip encoder length check for Whisper + + if model_config.is_multimodal_model: + suggestion = ( + "Make sure that `max_model_len` is no smaller than the " + "number of text tokens plus multimodal tokens. For image " + "inputs, the number of image tokens depends on the number " + "of images, and possibly their aspect ratios as well." + ) + else: + suggestion = ( + "Make sure that `max_model_len` is no smaller than the " + "number of text tokens." + ) + + raise ValueError( + f"The {prompt_type} prompt (length {prompt_len}) is " + f"longer than the maximum model length of {max_prompt_len}. " + f"{suggestion}" + ) + + # TODO: Find out how many placeholder tokens are there so we can + # check that chunked prefill does not truncate them + # max_batch_len = self.scheduler_config.max_num_batched_tokens + + if ( + prompt_len == max_prompt_len + and prompt_type == "decoder" + and not model_config.is_multimodal_model + and self.model_config.runner_type != "pooling" + ): + suggestion = ( + "Make sure that `max_model_len` is no smaller than the " + "number of text tokens (prompt + requested output tokens)." + ) + raise ValueError( + f"The {prompt_type} prompt (length {prompt_len}) plus the number of " + f"requested output tokens (at least 1) is longer than the maximum " + f"model length of {max_prompt_len}. {suggestion}" + ) + + def stat_mm_cache(self) -> MultiModalCacheStats | None: + return self.input_preprocessor.stat_mm_cache() + + def clear_mm_cache(self) -> None: + self.input_preprocessor.clear_mm_cache() diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index dffe05445ee4..7db038b6fcd2 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -24,7 +24,7 @@ from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask from vllm.tracing import init_tracer -from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs +from vllm.transformers_utils.tokenizer import TokenizerLike, init_tokenizer_from_configs from vllm.usage.usage_lib import UsageContext from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.core_client import EngineCoreClient @@ -95,9 +95,10 @@ def __init__( ) # OutputProcessor (convert EngineCoreOutputs --> RequestOutput). - stream_interval = self.vllm_config.scheduler_config.stream_interval self.output_processor = OutputProcessor( - self.tokenizer, log_stats=self.log_stats, stream_interval=stream_interval + self.tokenizer, + log_stats=self.log_stats, + stream_interval=self.vllm_config.scheduler_config.stream_interval, ) endpoint = self.observability_config.otlp_traces_endpoint if endpoint is not None: @@ -346,14 +347,14 @@ def get_metrics(self) -> list[Metric]: return get_metrics_snapshot() @property - def tokenizer(self) -> AnyTokenizer | None: + def tokenizer(self) -> TokenizerLike | None: return self.processor.tokenizer @tokenizer.setter - def tokenizer(self, tokenizer: AnyTokenizer | None) -> None: + def tokenizer(self, tokenizer: TokenizerLike | None) -> None: self.processor.tokenizer = tokenizer - def get_tokenizer(self) -> AnyTokenizer: + def get_tokenizer(self) -> TokenizerLike: if self.tokenizer is None: raise ValueError( "Unable to get tokenizer because skip_tokenizer_init is True" diff --git a/vllm/v1/engine/logprobs.py b/vllm/v1/engine/logprobs.py index 63064a2c65d6..1c8f808bc25b 100644 --- a/vllm/v1/engine/logprobs.py +++ b/vllm/v1/engine/logprobs.py @@ -13,7 +13,7 @@ create_sample_logprobs, ) from vllm.transformers_utils.detokenizer_utils import ( - AnyTokenizer, + TokenizerLike, convert_ids_list_to_tokens, ) from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest @@ -28,7 +28,7 @@ class LogprobsProcessor: # Tokenizer for this request, # None if detokenization is disabled. - tokenizer: AnyTokenizer | None + tokenizer: TokenizerLike | None # Logprobs for this request logprobs: SampleLogprobs | None @@ -40,7 +40,7 @@ class LogprobsProcessor: @classmethod def from_new_request( cls, - tokenizer: AnyTokenizer | None, + tokenizer: TokenizerLike | None, request: EngineCoreRequest, ) -> "LogprobsProcessor": sampling_params = request.sampling_params diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 0453c4a77f0c..7f8f7388c917 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -16,7 +16,7 @@ ) from vllm.sampling_params import RequestOutputKind from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason from vllm.v1.engine.detokenizer import IncrementalDetokenizer @@ -139,7 +139,7 @@ def __init__( @classmethod def from_new_request( cls, - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike | None, request: EngineCoreRequest, prompt: str | None, parent_req: ParentRequest | None, @@ -341,7 +341,10 @@ class OutputProcessor: """Process EngineCoreOutputs into RequestOutputs.""" def __init__( - self, tokenizer: AnyTokenizer, log_stats: bool, stream_interval: int = 1 + self, + tokenizer: TokenizerLike | None, + log_stats: bool, + stream_interval: int = 1, ): self.log_stats = log_stats self.tokenizer = tokenizer diff --git a/vllm/v1/engine/processor.py b/vllm/v1/engine/processor.py index af4f0e410e25..ffdd50f8a046 100644 --- a/vllm/v1/engine/processor.py +++ b/vllm/v1/engine/processor.py @@ -19,7 +19,7 @@ from vllm.multimodal.utils import argsort_mm_positions from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.v1.engine import EngineCoreRequest @@ -40,7 +40,7 @@ class Processor: def __init__( self, vllm_config: VllmConfig, - tokenizer: AnyTokenizer | None, + tokenizer: TokenizerLike | None, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, ) -> None: self.vllm_config = vllm_config @@ -62,11 +62,11 @@ def __init__( ) @property - def tokenizer(self) -> AnyTokenizer | None: + def tokenizer(self) -> TokenizerLike | None: return self.input_preprocessor.tokenizer @tokenizer.setter - def tokenizer(self, tokenizer: AnyTokenizer | None) -> None: + def tokenizer(self, tokenizer: TokenizerLike | None) -> None: self.input_preprocessor.tokenizer = tokenizer def _validate_logprobs( diff --git a/vllm/v1/structured_output/backend_types.py b/vllm/v1/structured_output/backend_types.py index 7dc9589b63b8..e28273bcf8b3 100644 --- a/vllm/v1/structured_output/backend_types.py +++ b/vllm/v1/structured_output/backend_types.py @@ -10,10 +10,10 @@ import torch from vllm.config import VllmConfig - from vllm.transformers_utils.tokenizer import AnyTokenizer + from vllm.transformers_utils.tokenizer import TokenizerLike else: VllmConfig = object - AnyTokenizer = object + TokenizerLike = object class StructuredOutputOptions(enum.Enum): @@ -100,7 +100,7 @@ class StructuredOutputBackend(ABC): """Engine-level backend for structured output requests.""" vllm_config: VllmConfig - tokenizer: AnyTokenizer + tokenizer: TokenizerLike vocab_size: int @abstractmethod diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index d2d14fcfc436..3d1aefdf888d 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -24,7 +24,7 @@ import transformers.models.gpt2.tokenization_gpt2 as tokenization_gpt2 import xgrammar as xgr - from vllm.transformers_utils.tokenizer import AnyTokenizer + from vllm.transformers_utils.tokenizer import TokenizerLike from vllm.v1.worker.gpu_input_batch import InputBatch else: xgr = LazyLoader("xgr", globals(), "xgrammar") @@ -36,7 +36,7 @@ "transformers.models.gpt2.tokenization_gpt2", ) - AnyTokenizer = object + TokenizerLike = object SchedulerOutput = object InputBatch = object @@ -195,7 +195,7 @@ def get_outlines_cache(): def _reduced_vocabulary( - tokenizer: AnyTokenizer, + tokenizer: TokenizerLike, eos_token_id: int, ) -> dict[bytes, list[int]]: """Create a map from vocabulary tokens to lists of equivalent token ids. @@ -222,7 +222,7 @@ def convert_token_to_string(token: str) -> str: vocabulary: dict[bytes, list[int]] = {} empty_token_ids: list[int] = [] for token, token_idx in tokenizer.get_vocab().items(): - if token in tokenizer.all_special_tokens: # type: ignore + if token in tokenizer.all_special_tokens: continue token_str = convert_token_to_string(token) @@ -261,7 +261,7 @@ def convert_token_to_string(token: str) -> str: return vocabulary -def get_outlines_vocabulary(tokenizer: AnyTokenizer) -> oc.Vocabulary: +def get_outlines_vocabulary(tokenizer: TokenizerLike) -> oc.Vocabulary: """Get the `Vocabulary` object for a given tokenizer.""" if hasattr(tokenizer, "_outlines_vocabulary"): return tokenizer._outlines_vocabulary # type: ignore From 668eb2ced87ca37ca523fe98a7a72e6ce179387c Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 17:22:09 +0000 Subject: [PATCH 02/31] BC Signed-off-by: DarkLight1337 --- vllm/transformers_utils/tokenizer.py | 2 +- vllm/transformers_utils/tokenizer_base.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index a7c16ab60da3..a6b14dacf217 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -37,7 +37,7 @@ def __getattr__(name: str): - # TODO: Move AnyTokenizer into this file + # TODO: Move TokenizerLike into this file # and move TokenizerRegistry into `registry.py` with a deprecation if name == "AnyTokenizer": warnings.warn( diff --git a/vllm/transformers_utils/tokenizer_base.py b/vllm/transformers_utils/tokenizer_base.py index cb7b76ee6155..3b81b09b58c8 100644 --- a/vllm/transformers_utils/tokenizer_base.py +++ b/vllm/transformers_utils/tokenizer_base.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib +import warnings from typing import TYPE_CHECKING, Any, Protocol from typing_extensions import Self, runtime_checkable @@ -10,6 +11,23 @@ from vllm.entrypoints.chat_utils import ChatCompletionMessageParam +def __getattr__(name: str): + # TODO: Move TokenizerLike into `tokenizer.py` + # and move TokenizerRegistry into `registry.py` with a deprecation + if name == "TokenizerBase": + warnings.warn( + "`vllm.transformers_utils.tokenizer_base.TokenizerBase` has been moved to " + "`vllm.transformers_utils.tokenizer.TokenizerLike`. " + "The old name will be removed in v0.13.", + DeprecationWarning, + stacklevel=2, + ) + + return TokenizerLike + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + + @runtime_checkable class TokenizerLike(Protocol): @classmethod From 6684cd0fde62d55091defd9739f1dd17022c9350 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 17:27:23 +0000 Subject: [PATCH 03/31] Unnecessary quote Signed-off-by: DarkLight1337 --- vllm/transformers_utils/tokenizer_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/transformers_utils/tokenizer_base.py b/vllm/transformers_utils/tokenizer_base.py index 3b81b09b58c8..c692b4beb6f5 100644 --- a/vllm/transformers_utils/tokenizer_base.py +++ b/vllm/transformers_utils/tokenizer_base.py @@ -37,7 +37,7 @@ def from_pretrained( /, *, revision: str | None = None, - ) -> "Self": + ) -> Self: raise NotImplementedError @property From 9f3ab67987c8a36b7b5e73cd7b6490de71c6e761 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 17:29:29 +0000 Subject: [PATCH 04/31] Rename Signed-off-by: DarkLight1337 --- tests/transformers_utils/{test_get_eos.py => test_config.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/transformers_utils/{test_get_eos.py => test_config.py} (100%) diff --git a/tests/transformers_utils/test_get_eos.py b/tests/transformers_utils/test_config.py similarity index 100% rename from tests/transformers_utils/test_get_eos.py rename to tests/transformers_utils/test_config.py From d35e431611803499e120b68a23b491057419d076 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 17:32:11 +0000 Subject: [PATCH 05/31] Forward merge Signed-off-by: DarkLight1337 --- vllm/transformers_utils/processor.py | 427 +-------------------------- 1 file changed, 12 insertions(+), 415 deletions(-) diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index 63cdf6337034..bc5c7fc400fd 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -1,423 +1,20 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import warnings -import importlib -import inspect -from functools import lru_cache -from typing import TYPE_CHECKING, Any, cast, get_args, get_type_hints -from transformers import ( - AutoFeatureExtractor, - AutoImageProcessor, - AutoProcessor, - AutoVideoProcessor, -) -from transformers.feature_extraction_utils import FeatureExtractionMixin -from transformers.image_processing_utils import BaseImageProcessor -from transformers.processing_utils import ProcessorMixin -from transformers.video_processing_utils import BaseVideoProcessor -from typing_extensions import TypeVar +def __getattr__(name: str): + if name == "Processor": + from .input_processor import InputProcessor -from vllm.transformers_utils.utils import convert_model_repo_to_path, is_gguf -from vllm.utils.func_utils import get_allowed_kwarg_only_overrides - -if TYPE_CHECKING: - from vllm.config import ModelConfig - -_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin) -_V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor) - - -class HashableDict(dict): - """ - A dictionary that can be hashed by lru_cache. - """ - - # NOTE: pythonic dict is not hashable, - # we override on it directly for simplicity - def __hash__(self) -> int: # type: ignore[override] - return hash(frozenset(self.items())) - - -class HashableList(list): - """ - A list that can be hashed by lru_cache. - """ - - def __hash__(self) -> int: # type: ignore[override] - return hash(tuple(self)) - - -def _get_processor_factory_fn(processor_cls: type | tuple[type, ...]): - if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin: - return AutoProcessor.from_pretrained - if hasattr(processor_cls, "from_pretrained"): - return processor_cls.from_pretrained - - return processor_cls - - -@lru_cache -def _collect_dynamic_keys_from_processing_kwargs(kwargs_cls: type) -> set[str]: - dynamic_kwargs: set[str] = set() - if kwargs_cls is None: - return dynamic_kwargs - # get kwargs annotations in processor - # merge text_kwargs / images_kwargs / videos_kwargs / audio_kwargs - kwargs_type_annotations = get_type_hints(kwargs_cls) - for kw_type in ("text_kwargs", "images_kwargs", "videos_kwargs", "audio_kwargs"): - if kw_type in kwargs_type_annotations: - kw_annotations = get_type_hints(kwargs_type_annotations[kw_type]) - for kw_name in kw_annotations: - dynamic_kwargs.add(kw_name) - dynamic_kwargs |= {"text_kwargs", "images_kwargs", "videos_kwargs", "audio_kwargs"} - return dynamic_kwargs - - -def _merge_mm_kwargs( - model_config: "ModelConfig", - processor_cls: type | tuple[type, ...], - /, - **kwargs, -): - mm_config = model_config.get_multimodal_config() - merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs) - - factory = _get_processor_factory_fn(processor_cls) - allowed_kwargs = get_allowed_kwarg_only_overrides( - factory, - merged_kwargs, - requires_kw_only=False, - allow_var_kwargs=True, - ) - # NOTE: Pythonic dict is not hashable and will raise unhashable type - # error when calling `cached_get_processor`, therefore we need to - # wrap it to a hashable dict. - for key, value in allowed_kwargs.items(): - if isinstance(value, dict): - allowed_kwargs[key] = HashableDict(value) - if isinstance(value, list): - allowed_kwargs[key] = HashableList(value) - - return allowed_kwargs - - -def get_processor( - processor_name: str, - *args: Any, - revision: str | None = None, - trust_remote_code: bool = False, - processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin, - **kwargs: Any, -) -> _P: - """Load a processor for the given model name via HuggingFace.""" - if revision is None: - revision = "main" - try: - processor_name = convert_model_repo_to_path(processor_name) - if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin: - processor = AutoProcessor.from_pretrained( - processor_name, - *args, - revision=revision, - trust_remote_code=trust_remote_code, - **kwargs, - ) - elif issubclass(processor_cls, ProcessorMixin): - processor = processor_cls.from_pretrained( - processor_name, - *args, - revision=revision, - trust_remote_code=trust_remote_code, - **kwargs, - ) - else: - # Processors that are standalone classes unrelated to HF - processor = processor_cls(*args, **kwargs) - except ValueError as e: - # If the error pertains to the processor class not existing or not - # currently being imported, suggest using the --trust-remote-code flag. - # Unlike AutoTokenizer, AutoProcessor does not separate such errors - if not trust_remote_code: - err_msg = ( - "Failed to load the processor. If the processor is " - "a custom processor not yet available in the HuggingFace " - "transformers library, consider setting " - "`trust_remote_code=True` in LLM or using the " - "`--trust-remote-code` flag in the CLI." - ) - raise RuntimeError(err_msg) from e - else: - raise e - - if not isinstance(processor, processor_cls): - raise TypeError( - "Invalid type of HuggingFace processor. " - f"Expected type: {processor_cls}, but " - f"found type: {type(processor)}" - ) - - return processor - - -cached_get_processor = lru_cache(get_processor) - - -@lru_cache -def get_processor_kwargs_from_processor(processor: _P) -> set[str]: - try: - # get kwargs annotations in processor - call_kwargs = inspect.signature(type(processor).__call__).parameters.get( - "kwargs" - ) - call_kwargs_annotations = call_kwargs.annotation if call_kwargs else None - # if the processor has explicit kwargs annotation, use it - if call_kwargs_annotations not in (None, inspect._empty): - # get_type_hints will parse all type annotations at runtime, - # and if an annotation refers to a type or - # name that hasn’t been imported or defined, it will raise an error. - # So we use __annotations__ to get the raw annotations directly. - return _collect_dynamic_keys_from_processing_kwargs( - get_args(call_kwargs_annotations)[0] - ) - # otherwise, try to get from ProcessingKwargs - else: - module_name = type(processor).__module__ - mod = importlib.import_module(module_name) - # find *ProcessingKwargs in the module - processor_kwargs: set[str] = set() - for name, obj in vars(mod).items(): - if name.endswith("ProcessingKwargs"): - processor_kwargs = ( - processor_kwargs - | _collect_dynamic_keys_from_processing_kwargs(obj) - ) - return processor_kwargs - except Exception: - return set() - - -def cached_get_processor_without_dynamic_kwargs( - processor_name: str, - *args: Any, - revision: str | None = None, - trust_remote_code: bool = False, - processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin, - **kwargs: Any, -) -> _P: - # Step 1: use default kwargs to get a temporary processor instance - processor = cached_get_processor( - processor_name, - revision=revision, - trust_remote_code=trust_remote_code, - processor_cls=processor_cls, # type: ignore[arg-type] - ) - - # Step 2: use temporary processor collect dynamic keys - dynamic_keys = get_processor_kwargs_from_processor(processor) - - # Step 3: use dynamic_keys filter kwargs - filtered_kwargs = {k: v for k, v in kwargs.items() if k not in dynamic_keys} - - # Step 4: use filtered kwargs to get final processor instance - final_processor = cached_get_processor( - processor_name, - revision=revision, - trust_remote_code=trust_remote_code, - processor_cls=processor_cls, # type: ignore[arg-type] - **filtered_kwargs, - ) - - return final_processor - - -def cached_processor_from_config( - model_config: "ModelConfig", - processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin, - **kwargs: Any, -) -> _P: - if is_gguf(model_config.model): - assert not is_gguf(model_config.tokenizer), ( - "For multimodal GGUF models, the original tokenizer " - "should be used to correctly load processor." + warnings.warn( + "`vllm.v1.engine.processor.Processor` has been moved to " + "`vllm.v1.engine.input_processor.InputProcessor`. " + "The old name will be removed in v0.13.", + DeprecationWarning, + stacklevel=2, ) - model = model_config.tokenizer - revision = model_config.tokenizer_revision - else: - model = model_config.model - revision = model_config.revision - - return cached_get_processor_without_dynamic_kwargs( - model, - revision=revision, - trust_remote_code=model_config.trust_remote_code, - processor_cls=processor_cls, # type: ignore[arg-type] - **_merge_mm_kwargs(model_config, processor_cls, **kwargs), - ) - - -def get_feature_extractor( - processor_name: str, - *args: Any, - revision: str | None = None, - trust_remote_code: bool = False, - **kwargs: Any, -): - """Load an audio feature extractor for the given model name - via HuggingFace.""" - try: - processor_name = convert_model_repo_to_path(processor_name) - feature_extractor = AutoFeatureExtractor.from_pretrained( - processor_name, - *args, - revision=revision, - trust_remote_code=trust_remote_code, - **kwargs, - ) - except ValueError as e: - # If the error pertains to the processor class not existing or not - # currently being imported, suggest using the --trust-remote-code flag. - # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors - if not trust_remote_code: - err_msg = ( - "Failed to load the feature extractor. If the feature " - "extractor is a custom extractor not yet available in the " - "HuggingFace transformers library, consider setting " - "`trust_remote_code=True` in LLM or using the " - "`--trust-remote-code` flag in the CLI." - ) - raise RuntimeError(err_msg) from e - else: - raise e - return cast(FeatureExtractionMixin, feature_extractor) - - -cached_get_feature_extractor = lru_cache(get_feature_extractor) - - -def cached_feature_extractor_from_config( - model_config: "ModelConfig", - **kwargs: Any, -): - return cached_get_feature_extractor( - model_config.model, - revision=model_config.revision, - trust_remote_code=model_config.trust_remote_code, - **_merge_mm_kwargs(model_config, AutoFeatureExtractor, **kwargs), - ) - - -def get_image_processor( - processor_name: str, - *args: Any, - revision: str | None = None, - trust_remote_code: bool = False, - **kwargs: Any, -): - """Load an image processor for the given model name via HuggingFace.""" - try: - processor_name = convert_model_repo_to_path(processor_name) - processor = AutoImageProcessor.from_pretrained( - processor_name, - *args, - revision=revision, - trust_remote_code=trust_remote_code, - **kwargs, - ) - except ValueError as e: - # If the error pertains to the processor class not existing or not - # currently being imported, suggest using the --trust-remote-code flag. - # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors - if not trust_remote_code: - err_msg = ( - "Failed to load the image processor. If the image processor is " - "a custom processor not yet available in the HuggingFace " - "transformers library, consider setting " - "`trust_remote_code=True` in LLM or using the " - "`--trust-remote-code` flag in the CLI." - ) - raise RuntimeError(err_msg) from e - else: - raise e - - return cast(BaseImageProcessor, processor) - - -cached_get_image_processor = lru_cache(get_image_processor) - - -def cached_image_processor_from_config( - model_config: "ModelConfig", - **kwargs: Any, -): - if is_gguf(model_config.model): - assert not is_gguf(model_config.tokenizer), ( - "For multimodal GGUF models, the original tokenizer " - "should be used to correctly load image processor." - ) - model = model_config.tokenizer - revision = model_config.tokenizer_revision - else: - model = model_config.model - revision = model_config.revision - return cached_get_image_processor( - model, - revision=revision, - trust_remote_code=model_config.trust_remote_code, - **_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs), - ) - - -def get_video_processor( - processor_name: str, - *args: Any, - revision: str | None = None, - trust_remote_code: bool = False, - processor_cls_overrides: type[_V] | None = None, - **kwargs: Any, -): - """Load a video processor for the given model name via HuggingFace.""" - try: - processor_name = convert_model_repo_to_path(processor_name) - processor_cls = processor_cls_overrides or AutoVideoProcessor - processor = processor_cls.from_pretrained( - processor_name, - *args, - revision=revision, - trust_remote_code=trust_remote_code, - **kwargs, - ) - except ValueError as e: - # If the error pertains to the processor class not existing or not - # currently being imported, suggest using the --trust-remote-code flag. - # Unlike AutoTokenizer, AutoVideoProcessor does not separate such errors - if not trust_remote_code: - err_msg = ( - "Failed to load the video processor. If the video processor is " - "a custom processor not yet available in the HuggingFace " - "transformers library, consider setting " - "`trust_remote_code=True` in LLM or using the " - "`--trust-remote-code` flag in the CLI." - ) - raise RuntimeError(err_msg) from e - else: - raise e - - return cast(BaseVideoProcessor, processor) - - -cached_get_video_processor = lru_cache(get_video_processor) + return InputProcessor -def cached_video_processor_from_config( - model_config: "ModelConfig", - processor_cls: type[_V] | None = None, - **kwargs: Any, -): - return cached_get_video_processor( - model_config.model, - revision=model_config.revision, - trust_remote_code=model_config.trust_remote_code, - processor_cls_overrides=processor_cls, # type: ignore[arg-type] - **_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs), - ) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") From a616c7a42f225613cb099f0d1c1dbc9579cc03b1 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 17:42:50 +0000 Subject: [PATCH 06/31] Oops Signed-off-by: DarkLight1337 --- vllm/transformers_utils/processor.py | 427 ++++++++++++++++++++++++++- 1 file changed, 415 insertions(+), 12 deletions(-) diff --git a/vllm/transformers_utils/processor.py b/vllm/transformers_utils/processor.py index bc5c7fc400fd..63cdf6337034 100644 --- a/vllm/transformers_utils/processor.py +++ b/vllm/transformers_utils/processor.py @@ -1,20 +1,423 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import warnings +import importlib +import inspect +from functools import lru_cache +from typing import TYPE_CHECKING, Any, cast, get_args, get_type_hints -def __getattr__(name: str): - if name == "Processor": - from .input_processor import InputProcessor +from transformers import ( + AutoFeatureExtractor, + AutoImageProcessor, + AutoProcessor, + AutoVideoProcessor, +) +from transformers.feature_extraction_utils import FeatureExtractionMixin +from transformers.image_processing_utils import BaseImageProcessor +from transformers.processing_utils import ProcessorMixin +from transformers.video_processing_utils import BaseVideoProcessor +from typing_extensions import TypeVar - warnings.warn( - "`vllm.v1.engine.processor.Processor` has been moved to " - "`vllm.v1.engine.input_processor.InputProcessor`. " - "The old name will be removed in v0.13.", - DeprecationWarning, - stacklevel=2, +from vllm.transformers_utils.utils import convert_model_repo_to_path, is_gguf +from vllm.utils.func_utils import get_allowed_kwarg_only_overrides + +if TYPE_CHECKING: + from vllm.config import ModelConfig + +_P = TypeVar("_P", bound=ProcessorMixin, default=ProcessorMixin) +_V = TypeVar("_V", bound=BaseVideoProcessor, default=BaseVideoProcessor) + + +class HashableDict(dict): + """ + A dictionary that can be hashed by lru_cache. + """ + + # NOTE: pythonic dict is not hashable, + # we override on it directly for simplicity + def __hash__(self) -> int: # type: ignore[override] + return hash(frozenset(self.items())) + + +class HashableList(list): + """ + A list that can be hashed by lru_cache. + """ + + def __hash__(self) -> int: # type: ignore[override] + return hash(tuple(self)) + + +def _get_processor_factory_fn(processor_cls: type | tuple[type, ...]): + if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin: + return AutoProcessor.from_pretrained + if hasattr(processor_cls, "from_pretrained"): + return processor_cls.from_pretrained + + return processor_cls + + +@lru_cache +def _collect_dynamic_keys_from_processing_kwargs(kwargs_cls: type) -> set[str]: + dynamic_kwargs: set[str] = set() + if kwargs_cls is None: + return dynamic_kwargs + # get kwargs annotations in processor + # merge text_kwargs / images_kwargs / videos_kwargs / audio_kwargs + kwargs_type_annotations = get_type_hints(kwargs_cls) + for kw_type in ("text_kwargs", "images_kwargs", "videos_kwargs", "audio_kwargs"): + if kw_type in kwargs_type_annotations: + kw_annotations = get_type_hints(kwargs_type_annotations[kw_type]) + for kw_name in kw_annotations: + dynamic_kwargs.add(kw_name) + dynamic_kwargs |= {"text_kwargs", "images_kwargs", "videos_kwargs", "audio_kwargs"} + return dynamic_kwargs + + +def _merge_mm_kwargs( + model_config: "ModelConfig", + processor_cls: type | tuple[type, ...], + /, + **kwargs, +): + mm_config = model_config.get_multimodal_config() + merged_kwargs = mm_config.merge_mm_processor_kwargs(kwargs) + + factory = _get_processor_factory_fn(processor_cls) + allowed_kwargs = get_allowed_kwarg_only_overrides( + factory, + merged_kwargs, + requires_kw_only=False, + allow_var_kwargs=True, + ) + # NOTE: Pythonic dict is not hashable and will raise unhashable type + # error when calling `cached_get_processor`, therefore we need to + # wrap it to a hashable dict. + for key, value in allowed_kwargs.items(): + if isinstance(value, dict): + allowed_kwargs[key] = HashableDict(value) + if isinstance(value, list): + allowed_kwargs[key] = HashableList(value) + + return allowed_kwargs + + +def get_processor( + processor_name: str, + *args: Any, + revision: str | None = None, + trust_remote_code: bool = False, + processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin, + **kwargs: Any, +) -> _P: + """Load a processor for the given model name via HuggingFace.""" + if revision is None: + revision = "main" + try: + processor_name = convert_model_repo_to_path(processor_name) + if isinstance(processor_cls, tuple) or processor_cls == ProcessorMixin: + processor = AutoProcessor.from_pretrained( + processor_name, + *args, + revision=revision, + trust_remote_code=trust_remote_code, + **kwargs, + ) + elif issubclass(processor_cls, ProcessorMixin): + processor = processor_cls.from_pretrained( + processor_name, + *args, + revision=revision, + trust_remote_code=trust_remote_code, + **kwargs, + ) + else: + # Processors that are standalone classes unrelated to HF + processor = processor_cls(*args, **kwargs) + except ValueError as e: + # If the error pertains to the processor class not existing or not + # currently being imported, suggest using the --trust-remote-code flag. + # Unlike AutoTokenizer, AutoProcessor does not separate such errors + if not trust_remote_code: + err_msg = ( + "Failed to load the processor. If the processor is " + "a custom processor not yet available in the HuggingFace " + "transformers library, consider setting " + "`trust_remote_code=True` in LLM or using the " + "`--trust-remote-code` flag in the CLI." + ) + raise RuntimeError(err_msg) from e + else: + raise e + + if not isinstance(processor, processor_cls): + raise TypeError( + "Invalid type of HuggingFace processor. " + f"Expected type: {processor_cls}, but " + f"found type: {type(processor)}" + ) + + return processor + + +cached_get_processor = lru_cache(get_processor) + + +@lru_cache +def get_processor_kwargs_from_processor(processor: _P) -> set[str]: + try: + # get kwargs annotations in processor + call_kwargs = inspect.signature(type(processor).__call__).parameters.get( + "kwargs" + ) + call_kwargs_annotations = call_kwargs.annotation if call_kwargs else None + # if the processor has explicit kwargs annotation, use it + if call_kwargs_annotations not in (None, inspect._empty): + # get_type_hints will parse all type annotations at runtime, + # and if an annotation refers to a type or + # name that hasn’t been imported or defined, it will raise an error. + # So we use __annotations__ to get the raw annotations directly. + return _collect_dynamic_keys_from_processing_kwargs( + get_args(call_kwargs_annotations)[0] + ) + # otherwise, try to get from ProcessingKwargs + else: + module_name = type(processor).__module__ + mod = importlib.import_module(module_name) + # find *ProcessingKwargs in the module + processor_kwargs: set[str] = set() + for name, obj in vars(mod).items(): + if name.endswith("ProcessingKwargs"): + processor_kwargs = ( + processor_kwargs + | _collect_dynamic_keys_from_processing_kwargs(obj) + ) + return processor_kwargs + except Exception: + return set() + + +def cached_get_processor_without_dynamic_kwargs( + processor_name: str, + *args: Any, + revision: str | None = None, + trust_remote_code: bool = False, + processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin, + **kwargs: Any, +) -> _P: + # Step 1: use default kwargs to get a temporary processor instance + processor = cached_get_processor( + processor_name, + revision=revision, + trust_remote_code=trust_remote_code, + processor_cls=processor_cls, # type: ignore[arg-type] + ) + + # Step 2: use temporary processor collect dynamic keys + dynamic_keys = get_processor_kwargs_from_processor(processor) + + # Step 3: use dynamic_keys filter kwargs + filtered_kwargs = {k: v for k, v in kwargs.items() if k not in dynamic_keys} + + # Step 4: use filtered kwargs to get final processor instance + final_processor = cached_get_processor( + processor_name, + revision=revision, + trust_remote_code=trust_remote_code, + processor_cls=processor_cls, # type: ignore[arg-type] + **filtered_kwargs, + ) + + return final_processor + + +def cached_processor_from_config( + model_config: "ModelConfig", + processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin, + **kwargs: Any, +) -> _P: + if is_gguf(model_config.model): + assert not is_gguf(model_config.tokenizer), ( + "For multimodal GGUF models, the original tokenizer " + "should be used to correctly load processor." ) + model = model_config.tokenizer + revision = model_config.tokenizer_revision + else: + model = model_config.model + revision = model_config.revision + + return cached_get_processor_without_dynamic_kwargs( + model, + revision=revision, + trust_remote_code=model_config.trust_remote_code, + processor_cls=processor_cls, # type: ignore[arg-type] + **_merge_mm_kwargs(model_config, processor_cls, **kwargs), + ) + + +def get_feature_extractor( + processor_name: str, + *args: Any, + revision: str | None = None, + trust_remote_code: bool = False, + **kwargs: Any, +): + """Load an audio feature extractor for the given model name + via HuggingFace.""" + try: + processor_name = convert_model_repo_to_path(processor_name) + feature_extractor = AutoFeatureExtractor.from_pretrained( + processor_name, + *args, + revision=revision, + trust_remote_code=trust_remote_code, + **kwargs, + ) + except ValueError as e: + # If the error pertains to the processor class not existing or not + # currently being imported, suggest using the --trust-remote-code flag. + # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors + if not trust_remote_code: + err_msg = ( + "Failed to load the feature extractor. If the feature " + "extractor is a custom extractor not yet available in the " + "HuggingFace transformers library, consider setting " + "`trust_remote_code=True` in LLM or using the " + "`--trust-remote-code` flag in the CLI." + ) + raise RuntimeError(err_msg) from e + else: + raise e + return cast(FeatureExtractionMixin, feature_extractor) + + +cached_get_feature_extractor = lru_cache(get_feature_extractor) + + +def cached_feature_extractor_from_config( + model_config: "ModelConfig", + **kwargs: Any, +): + return cached_get_feature_extractor( + model_config.model, + revision=model_config.revision, + trust_remote_code=model_config.trust_remote_code, + **_merge_mm_kwargs(model_config, AutoFeatureExtractor, **kwargs), + ) + + +def get_image_processor( + processor_name: str, + *args: Any, + revision: str | None = None, + trust_remote_code: bool = False, + **kwargs: Any, +): + """Load an image processor for the given model name via HuggingFace.""" + try: + processor_name = convert_model_repo_to_path(processor_name) + processor = AutoImageProcessor.from_pretrained( + processor_name, + *args, + revision=revision, + trust_remote_code=trust_remote_code, + **kwargs, + ) + except ValueError as e: + # If the error pertains to the processor class not existing or not + # currently being imported, suggest using the --trust-remote-code flag. + # Unlike AutoTokenizer, AutoImageProcessor does not separate such errors + if not trust_remote_code: + err_msg = ( + "Failed to load the image processor. If the image processor is " + "a custom processor not yet available in the HuggingFace " + "transformers library, consider setting " + "`trust_remote_code=True` in LLM or using the " + "`--trust-remote-code` flag in the CLI." + ) + raise RuntimeError(err_msg) from e + else: + raise e + + return cast(BaseImageProcessor, processor) + + +cached_get_image_processor = lru_cache(get_image_processor) + + +def cached_image_processor_from_config( + model_config: "ModelConfig", + **kwargs: Any, +): + if is_gguf(model_config.model): + assert not is_gguf(model_config.tokenizer), ( + "For multimodal GGUF models, the original tokenizer " + "should be used to correctly load image processor." + ) + model = model_config.tokenizer + revision = model_config.tokenizer_revision + else: + model = model_config.model + revision = model_config.revision + return cached_get_image_processor( + model, + revision=revision, + trust_remote_code=model_config.trust_remote_code, + **_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs), + ) + + +def get_video_processor( + processor_name: str, + *args: Any, + revision: str | None = None, + trust_remote_code: bool = False, + processor_cls_overrides: type[_V] | None = None, + **kwargs: Any, +): + """Load a video processor for the given model name via HuggingFace.""" + try: + processor_name = convert_model_repo_to_path(processor_name) + processor_cls = processor_cls_overrides or AutoVideoProcessor + processor = processor_cls.from_pretrained( + processor_name, + *args, + revision=revision, + trust_remote_code=trust_remote_code, + **kwargs, + ) + except ValueError as e: + # If the error pertains to the processor class not existing or not + # currently being imported, suggest using the --trust-remote-code flag. + # Unlike AutoTokenizer, AutoVideoProcessor does not separate such errors + if not trust_remote_code: + err_msg = ( + "Failed to load the video processor. If the video processor is " + "a custom processor not yet available in the HuggingFace " + "transformers library, consider setting " + "`trust_remote_code=True` in LLM or using the " + "`--trust-remote-code` flag in the CLI." + ) + raise RuntimeError(err_msg) from e + else: + raise e + + return cast(BaseVideoProcessor, processor) + + +cached_get_video_processor = lru_cache(get_video_processor) - return InputProcessor - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") +def cached_video_processor_from_config( + model_config: "ModelConfig", + processor_cls: type[_V] | None = None, + **kwargs: Any, +): + return cached_get_video_processor( + model_config.model, + revision=model_config.revision, + trust_remote_code=model_config.trust_remote_code, + processor_cls_overrides=processor_cls, # type: ignore[arg-type] + **_merge_mm_kwargs(model_config, AutoVideoProcessor, **kwargs), + ) From 94688c1f905df81e2a13fc0eb77f825c16a79de1 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 17:44:14 +0000 Subject: [PATCH 07/31] Docstring Signed-off-by: DarkLight1337 --- tests/transformers_utils/test_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/transformers_utils/test_config.py b/tests/transformers_utils/test_config.py index 921d77b1b335..7b56c9f0189d 100644 --- a/tests/transformers_utils/test_config.py +++ b/tests/transformers_utils/test_config.py @@ -3,7 +3,7 @@ """ This test file includes some cases where it is inappropriate to only get the `eos_token_id` from the tokenizer as defined by -{meth}`vllm.LLMEngine._get_eos_token_id`. +`vllm.LLMEngine._get_eos_token_id`. """ from vllm.transformers_utils.config import try_get_generation_config From 54787a5034d63742c6b78d1e980ec4c340928c68 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 17:46:36 +0000 Subject: [PATCH 08/31] Increase tolerance Signed-off-by: DarkLight1337 --- .buildkite/test-amd.yaml | 4 ++-- .buildkite/test-pipeline.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index b5758c9c4247..f41ce1eb6942 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -61,8 +61,8 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 4 mins - timeout_in_minutes: 10 +- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 10min + timeout_in_minutes: 15 mirror_hardwares: [amdexperimental, amdproduction] agent_pool: mi325_1 # grade: Blocking diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index f443146e2741..cfc74ae85ee4 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -57,8 +57,8 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 4 mins - timeout_in_minutes: 10 +- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 10min + timeout_in_minutes: 15 source_file_dependencies: - vllm/ - tests/test_inputs.py From bb46b1a1f254c95b8529f3a05510414cf0af75d1 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 23:28:36 +0000 Subject: [PATCH 09/31] [Bugfix] Fix wrong mock attribute Signed-off-by: DarkLight1337 --- tests/entrypoints/openai/test_serving_chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 492e15fc82a6..6a1b15c4131e 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -399,7 +399,7 @@ async def _fake_process_inputs( @dataclass class MockEngine: model_config: MockModelConfig = field(default_factory=MockModelConfig) - processor: MagicMock = field(default_factory=MagicMock) + input_processor: MagicMock = field(default_factory=MagicMock) io_processor: MagicMock = field(default_factory=MagicMock) From c8e948d229193dddb228b6a693855ab1bf427956 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 23:42:06 +0000 Subject: [PATCH 10/31] Avoid circular import Signed-off-by: DarkLight1337 --- vllm/transformers_utils/config.py | 7 ++++--- vllm/transformers_utils/tokenizer.py | 18 +++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 45c4358bbc8f..8f2cd3315ab9 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -26,8 +26,9 @@ from vllm import envs from vllm.logger import init_logger -from vllm.transformers_utils.config_parser_base import ConfigParserBase -from vllm.transformers_utils.repo_utils import ( + +from .config_parser_base import ConfigParserBase +from .repo_utils import ( _get_hf_token, file_or_path_exists, get_hf_file_to_dict, @@ -35,7 +36,7 @@ try_get_local_file, with_retry, ) -from vllm.transformers_utils.utils import ( +from .utils import ( check_gguf_file, is_gguf, is_remote_gguf, diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 81919a9ea98e..8248a32d5abb 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -16,17 +16,13 @@ from vllm import envs from vllm.logger import init_logger -from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config -from vllm.transformers_utils.gguf_utils import get_gguf_file_path_from_hf -from vllm.transformers_utils.registry import TokenizerRegistry -from vllm.transformers_utils.repo_utils import list_filtered_repo_files -from vllm.transformers_utils.tokenizers import MistralTokenizer -from vllm.transformers_utils.utils import ( - check_gguf_file, - is_gguf, - is_remote_gguf, - split_remote_gguf, -) + +from .config import get_sentence_transformer_tokenizer_config +from .gguf_utils import get_gguf_file_path_from_hf +from .registry import TokenizerRegistry +from .repo_utils import list_filtered_repo_files +from .tokenizers import MistralTokenizer +from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf if TYPE_CHECKING: from vllm.config import ModelConfig From 94b9c62c440c7417bb8d01ed993595216491cb47 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 23:50:57 +0000 Subject: [PATCH 11/31] Fix mypy Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 8953084c464d..9d204bcffcc4 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1079,7 +1079,7 @@ def _validate_chat_template( async def _preprocess_chat( self, request: ChatLikeRequest | ResponsesRequest, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, messages: list[ChatCompletionMessageParam], chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, @@ -1095,6 +1095,11 @@ async def _preprocess_chat( Sequence[RequestPrompt], list[EngineTokensPrompt], ]: + if tokenizer is None: + raise ValueError( + "Unable to get tokenizer because skip_tokenizer_init is True" + ) + model_config = self.model_config resolved_content_format = resolve_chat_template_content_format( From 3dbb92c4e12556eab1b21e689a940bdce7e6c5ef Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 23:56:41 +0000 Subject: [PATCH 12/31] Fix circular import Signed-off-by: DarkLight1337 --- vllm/transformers_utils/registry.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/vllm/transformers_utils/registry.py b/vllm/transformers_utils/registry.py index 348814ef5a7b..ee1f345228ea 100644 --- a/vllm/transformers_utils/registry.py +++ b/vllm/transformers_utils/registry.py @@ -1,8 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib +from typing import TYPE_CHECKING -from .tokenizer import TokenizerLike +if TYPE_CHECKING: + from .tokenizer import TokenizerLike class TokenizerRegistry: @@ -18,7 +20,7 @@ def get_tokenizer( tokenizer_name: str, *args, **kwargs, - ) -> TokenizerLike: + ) -> "TokenizerLike": tokenizer_cls = TokenizerRegistry.REGISTRY.get(tokenizer_name) if tokenizer_cls is None: raise ValueError(f"Tokenizer {tokenizer_name} not found.") From 9aeed950dd27a4eda3a020455923d1f212720368 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 28 Nov 2025 23:57:07 +0000 Subject: [PATCH 13/31] rel import Signed-off-by: DarkLight1337 --- vllm/transformers_utils/gguf_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/transformers_utils/gguf_utils.py b/vllm/transformers_utils/gguf_utils.py index c5b4d3f00090..cb1fc2d092e0 100644 --- a/vllm/transformers_utils/gguf_utils.py +++ b/vllm/transformers_utils/gguf_utils.py @@ -9,7 +9,8 @@ from transformers import Gemma3Config, PretrainedConfig, SiglipVisionConfig from vllm.logger import init_logger -from vllm.transformers_utils.repo_utils import list_filtered_repo_files + +from .repo_utils import list_filtered_repo_files logger = init_logger(__name__) From e18ea6b72d0519828da6c029c06eb154787ff9f8 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:00:09 +0000 Subject: [PATCH 14/31] Avoid circular import Signed-off-by: DarkLight1337 --- vllm/transformers_utils/tokenizer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 8248a32d5abb..385d11ed0a65 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -21,7 +21,6 @@ from .gguf_utils import get_gguf_file_path_from_hf from .registry import TokenizerRegistry from .repo_utils import list_filtered_repo_files -from .tokenizers import MistralTokenizer from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf if TYPE_CHECKING: @@ -314,6 +313,8 @@ def get_tokenizer( tokenizer: TokenizerLike if tokenizer_mode == "mistral": + from .tokenizers import MistralTokenizer + logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}") tokenizer = MistralTokenizer.from_pretrained( str(tokenizer_name), revision=revision From e90b9c14b97f0d588a5716d972753dd4389df39b Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:10:31 +0000 Subject: [PATCH 15/31] Move Signed-off-by: DarkLight1337 --- .../openai/tool_parsers/conftest.py | 2 +- .../tool_parsers/test_hermes_tool_parser.py | 2 +- .../test_llama3_json_tool_parser.py | 2 +- .../test_llama4_pythonic_tool_parser.py | 2 +- .../tool_parsers/test_olmo3_tool_parser.py | 2 +- .../tool_parsers/test_pythonic_tool_parser.py | 2 +- .../entrypoints/openai/tool_parsers/utils.py | 2 +- .../multimodal/generation/vlm_utils/core.py | 2 +- .../multimodal/generation/vlm_utils/types.py | 2 +- tests/multimodal/test_processing.py | 2 +- .../tool_use/test_ernie45_moe_tool_parser.py | 3 +- tests/tool_use/test_jamba_tool_parser.py | 3 +- tests/tool_use/test_qwen3coder_tool_parser.py | 3 +- tests/tool_use/test_seed_oss_tool_parser.py | 3 +- tests/tool_use/test_xlam_tool_parser.py | 3 +- .../test_cached_tokenizer.py | 3 +- tests/transformers_utils/test_tokenizer.py | 3 +- .../test_tokenizer_registry.py | 3 +- tests/v1/engine/test_output_processor.py | 2 +- vllm/benchmarks/datasets.py | 2 +- vllm/engine/protocol.py | 2 +- vllm/entrypoints/openai/serving_completion.py | 2 +- vllm/entrypoints/openai/serving_responses.py | 2 +- .../openai/serving_tokenization.py | 2 +- .../tool_parsers/abstract_tool_parser.py | 2 +- .../tool_parsers/deepseekv31_tool_parser.py | 2 +- .../tool_parsers/deepseekv3_tool_parser.py | 2 +- .../tool_parsers/ernie45_tool_parser.py | 2 +- .../tool_parsers/glm4_moe_tool_parser.py | 2 +- .../granite_20b_fc_tool_parser.py | 2 +- .../tool_parsers/granite_tool_parser.py | 2 +- .../tool_parsers/hunyuan_a13b_tool_parser.py | 2 +- .../tool_parsers/internlm2_tool_parser.py | 2 +- .../openai/tool_parsers/jamba_tool_parser.py | 3 +- .../tool_parsers/kimi_k2_tool_parser.py | 2 +- .../tool_parsers/longcat_tool_parser.py | 2 +- .../tool_parsers/minimax_m2_tool_parser.py | 2 +- .../tool_parsers/minimax_tool_parser.py | 2 +- .../openai/tool_parsers/openai_tool_parser.py | 2 +- .../tool_parsers/qwen3coder_tool_parser.py | 2 +- .../tool_parsers/qwen3xml_tool_parser.py | 2 +- .../tool_parsers/seed_oss_tool_parser.py | 2 +- .../openai/tool_parsers/step3_tool_parser.py | 2 +- .../openai/tool_parsers/xlam_tool_parser.py | 2 +- vllm/entrypoints/renderer.py | 2 +- vllm/inputs/preprocess.py | 2 +- vllm/logits_process.py | 2 +- vllm/model_executor/models/h2ovl.py | 2 +- vllm/model_executor/models/internvl.py | 2 +- vllm/model_executor/models/nemotron_vl.py | 2 +- vllm/model_executor/models/opencua.py | 2 +- vllm/model_executor/models/qwen2_vl.py | 2 +- vllm/model_executor/models/skyworkr1v.py | 2 +- vllm/model_executor/models/step3_vl.py | 2 +- vllm/reasoning/abs_reasoning_parsers.py | 2 +- vllm/reasoning/basic_parsers.py | 2 +- vllm/reasoning/minimax_m2_reasoning_parser.py | 2 +- vllm/reasoning/olmo3_reasoning_parser.py | 2 +- vllm/sampling_params.py | 2 +- vllm/transformers_utils/detokenizer_utils.py | 2 +- vllm/transformers_utils/registry.py | 2 +- vllm/transformers_utils/tokenizer.py | 116 ++---------------- vllm/transformers_utils/tokenizer_base.py | 4 +- .../transformers_utils/tokenizers/__init__.py | 2 + vllm/transformers_utils/tokenizers/mistral.py | 3 +- .../transformers_utils/tokenizers/protocol.py | 111 +++++++++++++++++ vllm/v1/engine/async_llm.py | 3 +- vllm/v1/engine/input_processor.py | 2 +- vllm/v1/engine/llm_engine.py | 3 +- vllm/v1/engine/output_processor.py | 2 +- vllm/v1/structured_output/backend_types.py | 2 +- vllm/v1/structured_output/utils.py | 2 +- 72 files changed, 205 insertions(+), 176 deletions(-) create mode 100644 vllm/transformers_utils/tokenizers/protocol.py diff --git a/tests/entrypoints/openai/tool_parsers/conftest.py b/tests/entrypoints/openai/tool_parsers/conftest.py index f22e71393d99..b748878a377d 100644 --- a/tests/entrypoints/openai/tool_parsers/conftest.py +++ b/tests/entrypoints/openai/tool_parsers/conftest.py @@ -4,7 +4,7 @@ import pytest from transformers import AutoTokenizer -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike @pytest.fixture(scope="function") diff --git a/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py index 65d05e3b8899..da4f35383aa2 100644 --- a/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py @@ -7,7 +7,7 @@ from vllm.entrypoints.openai.protocol import ChatCompletionRequest from vllm.entrypoints.openai.tool_parsers.hermes_tool_parser import Hermes2ProToolParser -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from ....utils import RemoteOpenAIServer diff --git a/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py index bd563de438cb..4d519f1943b6 100644 --- a/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py @@ -7,7 +7,7 @@ from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation from vllm.entrypoints.openai.tool_parsers.llama_tool_parser import Llama3JsonToolParser -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike @pytest.fixture diff --git a/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py index 5b67042264d4..a7c1856ca8c5 100644 --- a/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike # Test cases similar to pythonic parser but with Llama4 specific format SIMPLE_FUNCTION_OUTPUT = "[get_weather(city='LA', metric='C')]" diff --git a/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py index 3e559702b838..92fc68fe30ce 100644 --- a/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')" diff --git a/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py index 0382bc5bf035..fb3c8a62e21e 100644 --- a/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')" diff --git a/tests/entrypoints/openai/tool_parsers/utils.py b/tests/entrypoints/openai/tool_parsers/utils.py index 3ab77e986eac..3e7c39664454 100644 --- a/tests/entrypoints/openai/tool_parsers/utils.py +++ b/tests/entrypoints/openai/tool_parsers/utils.py @@ -11,7 +11,7 @@ ToolCall, ) from vllm.entrypoints.openai.tool_parsers import ToolParser -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike class StreamingToolReconstructor: diff --git a/tests/models/multimodal/generation/vlm_utils/core.py b/tests/models/multimodal/generation/vlm_utils/core.py index dfa109e5b605..5ea1c3ed5d99 100644 --- a/tests/models/multimodal/generation/vlm_utils/core.py +++ b/tests/models/multimodal/generation/vlm_utils/core.py @@ -9,7 +9,7 @@ from transformers.models.auto.auto_factory import _BaseAutoModelClass from vllm.config.model import RunnerOption -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from .....conftest import HfRunner, VllmRunner from ....registry import HF_EXAMPLE_MODELS diff --git a/tests/models/multimodal/generation/vlm_utils/types.py b/tests/models/multimodal/generation/vlm_utils/types.py index 4dd96b142143..261c26fff2d4 100644 --- a/tests/models/multimodal/generation/vlm_utils/types.py +++ b/tests/models/multimodal/generation/vlm_utils/types.py @@ -14,7 +14,7 @@ from vllm.config.model import RunnerOption from vllm.logprobs import SampleLogprobs -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from .....conftest import ( AUDIO_ASSETS, diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 86cae9abe048..7cc1af07d7c9 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -22,7 +22,7 @@ replace_token_matches, ) from vllm.multimodal.profiling import MultiModalProfiler -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from .utils import random_image diff --git a/tests/tool_use/test_ernie45_moe_tool_parser.py b/tests/tool_use/test_ernie45_moe_tool_parser.py index f199dcebe2f8..c7a090888a5f 100644 --- a/tests/tool_use/test_ernie45_moe_tool_parser.py +++ b/tests/tool_use/test_ernie45_moe_tool_parser.py @@ -15,7 +15,8 @@ ) from vllm.entrypoints.openai.tool_parsers.ernie45_tool_parser import Ernie45ToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizers import TokenizerLike # Use a common model that is likely to be available MODEL = "baidu/ERNIE-4.5-21B-A3B-Thinking" diff --git a/tests/tool_use/test_jamba_tool_parser.py b/tests/tool_use/test_jamba_tool_parser.py index 7ae9fc33d99a..572d5476b2d5 100644 --- a/tests/tool_use/test_jamba_tool_parser.py +++ b/tests/tool_use/test_jamba_tool_parser.py @@ -11,7 +11,8 @@ from vllm.entrypoints.openai.protocol import DeltaMessage, FunctionCall, ToolCall from vllm.entrypoints.openai.tool_parsers.jamba_tool_parser import JambaToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizers import TokenizerLike pytestmark = pytest.mark.cpu_test diff --git a/tests/tool_use/test_qwen3coder_tool_parser.py b/tests/tool_use/test_qwen3coder_tool_parser.py index a18a0c37909f..b3a65e0636d0 100644 --- a/tests/tool_use/test_qwen3coder_tool_parser.py +++ b/tests/tool_use/test_qwen3coder_tool_parser.py @@ -18,7 +18,8 @@ ) from vllm.entrypoints.openai.tool_parsers.qwen3xml_tool_parser import Qwen3XMLToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizers import TokenizerLike pytestmark = pytest.mark.cpu_test diff --git a/tests/tool_use/test_seed_oss_tool_parser.py b/tests/tool_use/test_seed_oss_tool_parser.py index fdcbfc75046c..855b9ec104c1 100644 --- a/tests/tool_use/test_seed_oss_tool_parser.py +++ b/tests/tool_use/test_seed_oss_tool_parser.py @@ -16,7 +16,8 @@ ) from vllm.entrypoints.openai.tool_parsers.seed_oss_tool_parser import SeedOssToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizers import TokenizerLike pytestmark = pytest.mark.cpu_test diff --git a/tests/tool_use/test_xlam_tool_parser.py b/tests/tool_use/test_xlam_tool_parser.py index 575f7f041d5a..9799f34af28a 100644 --- a/tests/tool_use/test_xlam_tool_parser.py +++ b/tests/tool_use/test_xlam_tool_parser.py @@ -14,7 +14,8 @@ ) from vllm.entrypoints.openai.tool_parsers.xlam_tool_parser import xLAMToolParser from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally -from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizers import TokenizerLike pytestmark = pytest.mark.cpu_test diff --git a/tests/transformers_utils/test_cached_tokenizer.py b/tests/transformers_utils/test_cached_tokenizer.py index 308faa5a4430..638feaa2bc33 100644 --- a/tests/transformers_utils/test_cached_tokenizer.py +++ b/tests/transformers_utils/test_cached_tokenizer.py @@ -6,7 +6,8 @@ import pytest from transformers import AutoTokenizer -from vllm.transformers_utils.tokenizer import TokenizerLike, get_cached_tokenizer +from vllm.transformers_utils.tokenizer import get_cached_tokenizer +from vllm.transformers_utils.tokenizers import TokenizerLike @pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"]) diff --git a/tests/transformers_utils/test_tokenizer.py b/tests/transformers_utils/test_tokenizer.py index b0bd70df56d9..ac0260913422 100644 --- a/tests/transformers_utils/test_tokenizer.py +++ b/tests/transformers_utils/test_tokenizer.py @@ -5,7 +5,8 @@ import pytest from transformers import PreTrainedTokenizerBase -from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizers import TokenizerLike def _get_missing_attrs(obj: object, target: type): diff --git a/tests/transformers_utils/test_tokenizer_registry.py b/tests/transformers_utils/test_tokenizer_registry.py index 567320c0cdbf..16b25940111d 100644 --- a/tests/transformers_utils/test_tokenizer_registry.py +++ b/tests/transformers_utils/test_tokenizer_registry.py @@ -1,7 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from vllm.transformers_utils.registry import TokenizerRegistry -from vllm.transformers_utils.tokenizer import TokenizerLike, get_tokenizer +from vllm.transformers_utils.tokenizer import get_tokenizer +from vllm.transformers_utils.tokenizers import TokenizerLike class TestTokenizer(TokenizerLike): diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 39d6bc385db2..13e59c58b45b 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -18,7 +18,7 @@ from vllm.lora.request import LoRARequest from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import RequestOutputKind, SamplingParams -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.v1.engine import ( EngineCoreEvent, EngineCoreEventType, diff --git a/vllm/benchmarks/datasets.py b/vllm/benchmarks/datasets.py index 37b0497c974b..2b5cd6e2f89f 100644 --- a/vllm/benchmarks/datasets.py +++ b/vllm/benchmarks/datasets.py @@ -39,7 +39,7 @@ from vllm.lora.utils import get_adapter_absolute_path from vllm.multimodal import MultiModalDataDict from vllm.multimodal.image import convert_image_mode -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.import_utils import PlaceholderModule try: diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index f994f1851290..6ff8f9392642 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -13,7 +13,7 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.input_processor import InputProcessor diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index f86c46308ba2..e96d92428e46 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -33,7 +33,7 @@ from vllm.logprobs import Logprob from vllm.outputs import RequestOutput from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.async_utils import merge_async_iterators from vllm.utils.collection_utils import as_list from vllm.v1.sample.logits_processor import validate_logits_processors_parameters diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index 103df2369f59..578e34bacfec 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -105,7 +105,7 @@ from vllm.logprobs import SampleLogprobs from vllm.outputs import CompletionOutput from vllm.sampling_params import SamplingParams, StructuredOutputsParams -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/serving_tokenization.py b/vllm/entrypoints/openai/serving_tokenization.py index a077e4d62fab..ec7c7a3e800a 100644 --- a/vllm/entrypoints/openai/serving_tokenization.py +++ b/vllm/entrypoints/openai/serving_tokenization.py @@ -22,7 +22,7 @@ from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.renderer import RenderConfig from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py index 9c917fe7a3b3..b69599ad3613 100644 --- a/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py @@ -22,7 +22,7 @@ from vllm.sampling_params import ( StructuredOutputsParams, ) -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.collection_utils import is_list_of from vllm.utils.import_utils import import_from_path diff --git a/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py index 5833bd116b84..37a9c8b4a735 100644 --- a/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py @@ -19,7 +19,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py index b249a733cbbc..fa9b1d7f9d1c 100644 --- a/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py @@ -19,7 +19,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py index 56a769ed65c0..aa301e7efaba 100644 --- a/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py @@ -19,7 +19,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py index 828b4a26b3d5..bdadd737bd7e 100644 --- a/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py @@ -22,7 +22,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py index bcc7ec1f86fb..0d872d058d94 100644 --- a/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py @@ -29,7 +29,7 @@ partial_json_loads, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py index 392f40a0f3b6..335352d1d6a0 100644 --- a/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py @@ -27,7 +27,7 @@ partial_json_loads, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py index 82f149bc0ce8..37176f55a81f 100644 --- a/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py @@ -22,7 +22,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import consume_space from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py index b2f73a7a5116..72fa8aab736a 100644 --- a/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py @@ -22,7 +22,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py index 71ce3432835f..9fd3edf5dd6f 100644 --- a/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py @@ -21,8 +21,7 @@ from vllm.entrypoints.openai.tool_parsers import ToolParser from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike -from vllm.transformers_utils.tokenizers import MistralTokenizer +from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py index 5933cff3f550..f7fe1168b9f7 100644 --- a/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py @@ -19,7 +19,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py index 57a790a59516..4ed053a56793 100644 --- a/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py @@ -4,7 +4,7 @@ import regex as re from vllm.entrypoints.openai.tool_parsers.hermes_tool_parser import Hermes2ProToolParser -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike class LongcatFlashToolParser(Hermes2ProToolParser): diff --git a/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py index 6117081d332e..3062eae87ae9 100644 --- a/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py @@ -21,7 +21,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py index 9fb43e66b56d..bb803c0beef8 100644 --- a/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py @@ -22,7 +22,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py index 03eda6f1a539..2e927f389250 100644 --- a/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py @@ -18,7 +18,7 @@ from vllm.logger import init_logger if TYPE_CHECKING: - from vllm.transformers_utils.tokenizer import TokenizerLike + from vllm.transformers_utils.tokenizers import TokenizerLike else: TokenizerLike = object diff --git a/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py index 5c054819322a..6f1478023edd 100644 --- a/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py @@ -22,7 +22,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py index ccfffc91b481..dc12ebc8daec 100644 --- a/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py @@ -23,7 +23,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py index bf15b69494e0..7e3bfee3bc5f 100644 --- a/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py @@ -25,7 +25,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py index e5ea6dcaf007..439327a09219 100644 --- a/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py @@ -21,7 +21,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py index d50d6937f6b8..7bd4541cf155 100644 --- a/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py @@ -21,7 +21,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) diff --git a/vllm/entrypoints/renderer.py b/vllm/entrypoints/renderer.py index 8f5a05c3707d..25caeae2d8ff 100644 --- a/vllm/entrypoints/renderer.py +++ b/vllm/entrypoints/renderer.py @@ -16,7 +16,7 @@ from vllm.inputs.data import TextPrompt as EngineTextPrompt from vllm.inputs.data import TokensPrompt as EngineTokensPrompt from vllm.inputs.parse import get_prompt_components, parse_raw_prompts -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.async_utils import AsyncMicrobatchTokenizer diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py index 47897083e5ff..b803a6f7f6ec 100644 --- a/vllm/inputs/preprocess.py +++ b/vllm/inputs/preprocess.py @@ -17,7 +17,7 @@ MultiModalUUIDDict, ) from vllm.multimodal.processing import BaseMultiModalProcessor -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.jsontree import json_iter_leaves from vllm.v1.metrics.stats import MultiModalCacheStats diff --git a/vllm/logits_process.py b/vllm/logits_process.py index b4f8c06a3a61..d167a899c28e 100644 --- a/vllm/logits_process.py +++ b/vllm/logits_process.py @@ -5,7 +5,7 @@ import torch -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike LogitsProcessor: TypeAlias = ( Callable[[list[int], torch.Tensor], torch.Tensor] diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index cec04dc01134..651b3143f337 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -28,7 +28,7 @@ PromptUpdate, PromptUpdateDetails, ) -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from .intern_vit import InternVisionModel from .internvl import ( diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index 884c92076900..8f52b58838e0 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -50,7 +50,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import set_default_torch_num_threads diff --git a/vllm/model_executor/models/nemotron_vl.py b/vllm/model_executor/models/nemotron_vl.py index 262cfb8836fa..a80a61455f45 100644 --- a/vllm/model_executor/models/nemotron_vl.py +++ b/vllm/model_executor/models/nemotron_vl.py @@ -35,7 +35,7 @@ from vllm.multimodal.processing import PromptUpdateDetails from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import cached_image_processor_from_config -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from .interfaces import ( MultiModalEmbeddings, diff --git a/vllm/model_executor/models/opencua.py b/vllm/model_executor/models/opencua.py index ecff5572757a..443b51e10b38 100644 --- a/vllm/model_executor/models/opencua.py +++ b/vllm/model_executor/models/opencua.py @@ -31,7 +31,7 @@ PromptReplacement, PromptUpdate, ) -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from .qwen2_5_vl import ( Qwen2_5_VisionTransformer as OpenCUAVisionTransformer, diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 0440017a984e..1dc0af7a483c 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -91,7 +91,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import ( diff --git a/vllm/model_executor/models/skyworkr1v.py b/vllm/model_executor/models/skyworkr1v.py index 1f77f3ffdaa2..827413cafe3d 100644 --- a/vllm/model_executor/models/skyworkr1v.py +++ b/vllm/model_executor/models/skyworkr1v.py @@ -47,7 +47,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP diff --git a/vllm/model_executor/models/step3_vl.py b/vllm/model_executor/models/step3_vl.py index 158a8fb21815..70a82cce4b82 100644 --- a/vllm/model_executor/models/step3_vl.py +++ b/vllm/model_executor/models/step3_vl.py @@ -44,7 +44,7 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs import Step3VisionEncoderConfig -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP diff --git a/vllm/reasoning/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py index f15c585b4aec..fb91885f6d5e 100644 --- a/vllm/reasoning/abs_reasoning_parsers.py +++ b/vllm/reasoning/abs_reasoning_parsers.py @@ -19,7 +19,7 @@ DeltaMessage, ResponsesRequest, ) - from vllm.transformers_utils.tokenizer import TokenizerLike + from vllm.transformers_utils.tokenizers import TokenizerLike else: ChatCompletionRequest = Any DeltaMessage = Any diff --git a/vllm/reasoning/basic_parsers.py b/vllm/reasoning/basic_parsers.py index 2af4eaf5f051..803696eea6b9 100644 --- a/vllm/reasoning/basic_parsers.py +++ b/vllm/reasoning/basic_parsers.py @@ -7,7 +7,7 @@ from vllm.entrypoints.openai.protocol import DeltaMessage from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike if TYPE_CHECKING: from vllm.entrypoints.openai.protocol import ( diff --git a/vllm/reasoning/minimax_m2_reasoning_parser.py b/vllm/reasoning/minimax_m2_reasoning_parser.py index f40f4570952a..4bd932074361 100644 --- a/vllm/reasoning/minimax_m2_reasoning_parser.py +++ b/vllm/reasoning/minimax_m2_reasoning_parser.py @@ -11,7 +11,7 @@ from vllm.logger import init_logger from vllm.reasoning.abs_reasoning_parsers import ReasoningParser from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/reasoning/olmo3_reasoning_parser.py b/vllm/reasoning/olmo3_reasoning_parser.py index 2e9e91c148a5..baa1d8355637 100644 --- a/vllm/reasoning/olmo3_reasoning_parser.py +++ b/vllm/reasoning/olmo3_reasoning_parser.py @@ -9,7 +9,7 @@ import regex as re if TYPE_CHECKING: - from vllm.transformers_utils.tokenizer import TokenizerLike + from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 9f793ada4f30..5210653bc40f 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -13,7 +13,7 @@ from vllm.logger import init_logger from vllm.logits_process import LogitsProcessor -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.v1.serial_utils import PydanticMsgspecMixin logger = init_logger(__name__) diff --git a/vllm/transformers_utils/detokenizer_utils.py b/vllm/transformers_utils/detokenizer_utils.py index 33d7fbacd66c..57947c31d97f 100644 --- a/vllm/transformers_utils/detokenizer_utils.py +++ b/vllm/transformers_utils/detokenizer_utils.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from .tokenizer import TokenizerLike +from .tokenizers import TokenizerLike def _replace_none_with_empty(tokens: list[str | None]): diff --git a/vllm/transformers_utils/registry.py b/vllm/transformers_utils/registry.py index ee1f345228ea..420b61f72b2b 100644 --- a/vllm/transformers_utils/registry.py +++ b/vllm/transformers_utils/registry.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from .tokenizer import TokenizerLike + from .tokenizers import TokenizerLike class TokenizerRegistry: diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 385d11ed0a65..0a5b4f61bfe4 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -8,11 +8,11 @@ import warnings from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Any, Protocol +from typing import TYPE_CHECKING, Any import huggingface_hub from transformers import AutoTokenizer, PreTrainedTokenizerBase -from typing_extensions import Self, assert_never, runtime_checkable +from typing_extensions import assert_never from vllm import envs from vllm.logger import init_logger @@ -25,7 +25,8 @@ if TYPE_CHECKING: from vllm.config import ModelConfig - from vllm.entrypoints.chat_utils import ChatCompletionMessageParam + + from .tokenizers import TokenizerLike logger = init_logger(__name__) @@ -33,9 +34,11 @@ def __getattr__(name: str): if name == "AnyTokenizer": + from .tokenizers import TokenizerLike + warnings.warn( "`vllm.transformers_utils.tokenizer.AnyTokenizer` has been renamed to " - "`vllm.transformers_utils.tokenizer.TokenizerLike`. " + "`vllm.transformers_utils.tokenizers.TokenizerLike`. " "The old name will be removed in v0.13.", DeprecationWarning, stacklevel=2, @@ -46,105 +49,8 @@ def __getattr__(name: str): raise AttributeError(f"module {__name__!r} has no attribute {name!r}") -@runtime_checkable -class TokenizerLike(Protocol): - @classmethod - def from_pretrained( - cls, - pretrained_model_name_or_path: str, - /, - *, - revision: str | None = None, - ) -> Self: - raise NotImplementedError - - @property - def all_special_tokens(self) -> list[str]: - raise NotImplementedError - - @property - def all_special_ids(self) -> list[int]: - raise NotImplementedError - - @property - def bos_token_id(self) -> int: - raise NotImplementedError - - @property - def eos_token_id(self) -> int: - raise NotImplementedError - - @property - def is_fast(self) -> bool: - raise NotImplementedError - - @property - def vocab_size(self) -> int: - raise NotImplementedError - - @property - def max_token_id(self) -> int: - raise NotImplementedError - - @property - def truncation_side(self) -> str: - raise NotImplementedError - - def __hash__(self) -> int: - return hash(id(self)) - - def __len__(self) -> int: - return self.vocab_size - - def __call__( - self, - text: str | list[str] | list[int], - text_pair: str | None = None, - add_special_tokens: bool = False, - truncation: bool = False, - max_length: int | None = None, - ): - raise NotImplementedError - - def get_vocab(self) -> dict[str, int]: - raise NotImplementedError - - def get_added_vocab(self) -> dict[str, int]: - raise NotImplementedError - - def encode( - self, - text: str, - truncation: bool | None = None, - max_length: int | None = None, - add_special_tokens: bool | None = None, - ) -> list[int]: - raise NotImplementedError - - def apply_chat_template( - self, - messages: list["ChatCompletionMessageParam"], - tools: list[dict[str, Any]] | None = None, - **kwargs, - ) -> list[int]: - raise NotImplementedError - - def convert_tokens_to_string(self, tokens: list[str]) -> str: - raise NotImplementedError - - def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str: - raise NotImplementedError - - def convert_ids_to_tokens( - self, - ids: list[int], - skip_special_tokens: bool = True, - ) -> list[str]: - raise NotImplementedError - - def decode_tokens( - tokenizer: TokenizerLike, + tokenizer: "TokenizerLike", token_ids: list[int], *, skip_special_tokens: bool | None = None, @@ -163,7 +69,7 @@ def decode_tokens( def encode_tokens( - tokenizer: TokenizerLike, + tokenizer: "TokenizerLike", text: str, *, truncation: bool | None = None, @@ -191,7 +97,7 @@ def encode_tokens( return tokenizer.encode(text, **kw_args) -def get_cached_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike: +def get_cached_tokenizer(tokenizer: "TokenizerLike") -> "TokenizerLike": """ By default, transformers will recompute multiple tokenizer properties each time they are called, leading to a significant slowdown. @@ -249,7 +155,7 @@ def get_tokenizer( revision: str | None = None, download_dir: str | None = None, **kwargs, -) -> TokenizerLike: +) -> "TokenizerLike": """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" if envs.VLLM_USE_MODELSCOPE: # download model from ModelScope hub, diff --git a/vllm/transformers_utils/tokenizer_base.py b/vllm/transformers_utils/tokenizer_base.py index 9e0c2a20e148..5bebadd29609 100644 --- a/vllm/transformers_utils/tokenizer_base.py +++ b/vllm/transformers_utils/tokenizer_base.py @@ -6,11 +6,11 @@ def __getattr__(name: str): if name == "TokenizerBase": - from .tokenizer import TokenizerLike + from .tokenizers import TokenizerLike warnings.warn( "`vllm.transformers_utils.tokenizer_base.TokenizerBase` " - "has been moved to `vllm.transformers_utils.tokenizer.TokenizerLike`. " + "has been moved to `vllm.transformers_utils.tokenizers.TokenizerLike`. " "The old name will be removed in v0.13.", DeprecationWarning, stacklevel=2, diff --git a/vllm/transformers_utils/tokenizers/__init__.py b/vllm/transformers_utils/tokenizers/__init__.py index b63cb26af46d..506ffb5dd597 100644 --- a/vllm/transformers_utils/tokenizers/__init__.py +++ b/vllm/transformers_utils/tokenizers/__init__.py @@ -7,8 +7,10 @@ truncate_tool_call_ids, validate_request_params, ) +from .protocol import TokenizerLike __all__ = [ + "TokenizerLike", "MistralTokenizer", "maybe_serialize_tool_calls", "truncate_tool_call_ids", diff --git a/vllm/transformers_utils/tokenizers/mistral.py b/vllm/transformers_utils/tokenizers/mistral.py index 2b31da444615..a42fb0e1e5f1 100644 --- a/vllm/transformers_utils/tokenizers/mistral.py +++ b/vllm/transformers_utils/tokenizers/mistral.py @@ -4,7 +4,8 @@ from typing import TYPE_CHECKING, Any, cast from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import TokenizerLike + +from .protocol import TokenizerLike if TYPE_CHECKING: from mistral_common.protocol.instruct.request import ( diff --git a/vllm/transformers_utils/tokenizers/protocol.py b/vllm/transformers_utils/tokenizers/protocol.py new file mode 100644 index 000000000000..4b0234614018 --- /dev/null +++ b/vllm/transformers_utils/tokenizers/protocol.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import TYPE_CHECKING, Any, Protocol + +from typing_extensions import Self, runtime_checkable + +from vllm.logger import init_logger + +if TYPE_CHECKING: + from vllm.entrypoints.chat_utils import ChatCompletionMessageParam + + +logger = init_logger(__name__) + + +@runtime_checkable +class TokenizerLike(Protocol): + @classmethod + def from_pretrained( + cls, + pretrained_model_name_or_path: str, + /, + *, + revision: str | None = None, + ) -> Self: + raise NotImplementedError + + @property + def all_special_tokens(self) -> list[str]: + raise NotImplementedError + + @property + def all_special_ids(self) -> list[int]: + raise NotImplementedError + + @property + def bos_token_id(self) -> int: + raise NotImplementedError + + @property + def eos_token_id(self) -> int: + raise NotImplementedError + + @property + def is_fast(self) -> bool: + raise NotImplementedError + + @property + def vocab_size(self) -> int: + raise NotImplementedError + + @property + def max_token_id(self) -> int: + raise NotImplementedError + + @property + def truncation_side(self) -> str: + raise NotImplementedError + + def __hash__(self) -> int: + return hash(id(self)) + + def __len__(self) -> int: + return self.vocab_size + + def __call__( + self, + text: str | list[str] | list[int], + text_pair: str | None = None, + add_special_tokens: bool = False, + truncation: bool = False, + max_length: int | None = None, + ): + raise NotImplementedError + + def get_vocab(self) -> dict[str, int]: + raise NotImplementedError + + def get_added_vocab(self) -> dict[str, int]: + raise NotImplementedError + + def encode( + self, + text: str, + truncation: bool | None = None, + max_length: int | None = None, + add_special_tokens: bool | None = None, + ) -> list[int]: + raise NotImplementedError + + def apply_chat_template( + self, + messages: list["ChatCompletionMessageParam"], + tools: list[dict[str, Any]] | None = None, + **kwargs, + ) -> list[int]: + raise NotImplementedError + + def convert_tokens_to_string(self, tokens: list[str]) -> str: + raise NotImplementedError + + def decode(self, ids: list[int] | int, skip_special_tokens: bool = True) -> str: + raise NotImplementedError + + def convert_ids_to_tokens( + self, + ids: list[int], + skip_special_tokens: bool = True, + ) -> list[str]: + raise NotImplementedError diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index d7a7253d15b6..77d4d0496748 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -28,7 +28,8 @@ from vllm.tasks import SupportedTask from vllm.tracing import init_tracer from vllm.transformers_utils.config import maybe_register_config_serialize_by_value -from vllm.transformers_utils.tokenizer import TokenizerLike, init_tokenizer_from_configs +from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.usage.usage_lib import UsageContext from vllm.utils.async_utils import cancel_task_threadsafe from vllm.utils.collection_utils import as_list diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 6fac7f5118ea..7c67d2421b04 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -19,7 +19,7 @@ from vllm.multimodal.utils import argsort_mm_positions from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.v1.engine import EngineCoreRequest diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 6c79d6f12177..89d02e8e14d8 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -24,7 +24,8 @@ from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask from vllm.tracing import init_tracer -from vllm.transformers_utils.tokenizer import TokenizerLike, init_tokenizer_from_configs +from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.usage.usage_lib import UsageContext from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.core_client import EngineCoreClient diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index 7f8f7388c917..f6308d63e82c 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -16,7 +16,7 @@ ) from vllm.sampling_params import RequestOutputKind from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context -from vllm.transformers_utils.tokenizer import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason from vllm.v1.engine.detokenizer import IncrementalDetokenizer diff --git a/vllm/v1/structured_output/backend_types.py b/vllm/v1/structured_output/backend_types.py index e28273bcf8b3..230b47470d6b 100644 --- a/vllm/v1/structured_output/backend_types.py +++ b/vllm/v1/structured_output/backend_types.py @@ -10,7 +10,7 @@ import torch from vllm.config import VllmConfig - from vllm.transformers_utils.tokenizer import TokenizerLike + from vllm.transformers_utils.tokenizers import TokenizerLike else: VllmConfig = object TokenizerLike = object diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 3d1aefdf888d..4602205e5aa6 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -24,7 +24,7 @@ import transformers.models.gpt2.tokenization_gpt2 as tokenization_gpt2 import xgrammar as xgr - from vllm.transformers_utils.tokenizer import TokenizerLike + from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.v1.worker.gpu_input_batch import InputBatch else: xgr = LazyLoader("xgr", globals(), "xgrammar") From 18710e39314697ea467e180f5d9042a3a6eaf8bc Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:16:13 +0000 Subject: [PATCH 16/31] Reword Signed-off-by: DarkLight1337 --- vllm/transformers_utils/tokenizer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 0a5b4f61bfe4..8fdd682e021b 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -37,7 +37,7 @@ def __getattr__(name: str): from .tokenizers import TokenizerLike warnings.warn( - "`vllm.transformers_utils.tokenizer.AnyTokenizer` has been renamed to " + "`vllm.transformers_utils.tokenizer.AnyTokenizer` has been moved to " "`vllm.transformers_utils.tokenizers.TokenizerLike`. " "The old name will be removed in v0.13.", DeprecationWarning, From f8577a52d8bb713ce5afc15841d82506b2755ca8 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:20:24 +0000 Subject: [PATCH 17/31] Fix imports Signed-off-by: DarkLight1337 --- .../models/multimodal/processing/test_common.py | 2 +- vllm/entrypoints/llm.py | 7 ++----- vllm/entrypoints/score_utils.py | 16 +++++++--------- vllm/model_executor/models/nano_nemotron_vl.py | 2 +- vllm/model_executor/models/pixtral.py | 6 ++---- vllm/model_executor/models/voxtral.py | 6 ++---- vllm/multimodal/processing.py | 7 ++----- vllm/multimodal/registry.py | 6 ++---- 8 files changed, 19 insertions(+), 33 deletions(-) diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 9638791ab5ca..4041f5af9ceb 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -23,10 +23,10 @@ from vllm.multimodal.inputs import MultiModalInputs from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext from vllm.transformers_utils.tokenizer import ( - MistralTokenizer, cached_tokenizer_from_config, encode_tokens, ) +from vllm.transformers_utils.tokenizers import MistralTokenizer from ....multimodal.utils import random_audio, random_image, random_video from ...registry import ( diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index d8a9279e94ee..37060d30b717 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -71,11 +71,8 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams from vllm.tasks import PoolingTask -from vllm.transformers_utils.tokenizer import ( - MistralTokenizer, - TokenizerLike, - get_cached_tokenizer, -) +from vllm.transformers_utils.tokenizer import get_cached_tokenizer +from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike from vllm.usage.usage_lib import UsageContext from vllm.utils.collection_utils import as_iter, is_list_of from vllm.utils.counter import Counter diff --git a/vllm/entrypoints/score_utils.py b/vllm/entrypoints/score_utils.py index 87f698efce24..afc7bf8c3354 100644 --- a/vllm/entrypoints/score_utils.py +++ b/vllm/entrypoints/score_utils.py @@ -19,11 +19,7 @@ from vllm.model_executor.models.interfaces import supports_score_template from vllm.multimodal.inputs import MultiModalDataDict from vllm.outputs import PoolingRequestOutput -from vllm.transformers_utils.tokenizer import ( - PreTrainedTokenizer, - PreTrainedTokenizerFast, - TokenizerLike, -) +from vllm.transformers_utils.tokenizer import TokenizerLike ScoreContentPartParam: TypeAlias = ( ChatCompletionContentPartImageParam | ChatCompletionContentPartImageEmbedsParam @@ -45,7 +41,7 @@ class ScoreMultiModalParam(TypedDict, total=False): def _cosine_similarity( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, + tokenizer: TokenizerLike, embed_1: list[PoolingRequestOutput], embed_2: list[PoolingRequestOutput], ) -> list[PoolingRequestOutput]: @@ -56,7 +52,7 @@ def _cosine_similarity( pair_score = scorer(emb_1.outputs.data, emb_2.outputs.data) padding = [] - if (pad_token_id := getattr(tokenizer, "pad_token_id", None)) is not None: + if (pad_token_id := tokenizer.pad_token_id) is not None: padding = [pad_token_id] tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids @@ -118,12 +114,14 @@ def _parse_score_content( mm_tracker: BaseMultiModalItemTracker, ) -> _ContentPart | None: if isinstance(data, str): - data = ChatCompletionContentPartTextParam(type="text", text=data) + part = ChatCompletionContentPartTextParam(type="text", text=data) + else: + part = data mm_parser = mm_tracker.create_parser() parse_res = _parse_chat_message_content_part( - data, + part, mm_parser, wrap_dicts=False, interleave_strings=False, diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 5177174a2d03..6ce7927b4568 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -75,10 +75,10 @@ from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.radio import RadioConfig from vllm.transformers_utils.tokenizer import ( - TokenizerLike, cached_tokenizer_from_config, encode_tokens, ) +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .utils import _merge_multimodal_embeddings diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index 3464de472add..069a6d2ab6d9 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -59,10 +59,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizer import ( - MistralTokenizer, - cached_tokenizer_from_config, -) +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config +from vllm.transformers_utils.tokenizers import MistralTokenizer from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 18ad8851fccd..427f3be5a263 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -51,10 +51,8 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizer import ( - MistralTokenizer, - cached_tokenizer_from_config, -) +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config +from vllm.transformers_utils.tokenizers import MistralTokenizer from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription from .utils import init_vllm_registered_model, maybe_prefix diff --git a/vllm/multimodal/processing.py b/vllm/multimodal/processing.py index 3fbc06765482..ea86f5f095d8 100644 --- a/vllm/multimodal/processing.py +++ b/vllm/multimodal/processing.py @@ -24,11 +24,8 @@ from vllm.logger import init_logger from vllm.transformers_utils.processor import cached_processor_from_config -from vllm.transformers_utils.tokenizer import ( - TokenizerLike, - decode_tokens, - encode_tokens, -) +from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens +from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.collection_utils import flatten_2d_lists, full_groupby from vllm.utils.func_utils import get_allowed_kwarg_only_overrides from vllm.utils.jsontree import JSONTree, json_map_leaves diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py index a6b7e0e09840..ed1a8bce5f0e 100644 --- a/vllm/multimodal/registry.py +++ b/vllm/multimodal/registry.py @@ -6,10 +6,8 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import ( - TokenizerLike, - cached_tokenizer_from_config, -) +from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config +from vllm.transformers_utils.tokenizers import TokenizerLike from .cache import BaseMultiModalProcessorCache from .processing import ( From 6c46abfd14dfb93d1fb886bde2f9b71e4729acf6 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:27:18 +0000 Subject: [PATCH 18/31] More updates Signed-off-by: DarkLight1337 --- benchmarks/backend_request_func.py | 2 +- tests/models/language/generation/test_mistral.py | 2 +- tests/models/multimodal/generation/test_voxtral.py | 2 +- vllm/entrypoints/chat_utils.py | 2 +- vllm/entrypoints/openai/serving_chat.py | 3 ++- vllm/entrypoints/openai/serving_engine.py | 2 +- vllm/entrypoints/openai/serving_score.py | 2 +- vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py | 6 +++--- vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py | 2 +- 9 files changed, 12 insertions(+), 11 deletions(-) diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py index 4021fede7215..f08ac66b9bc5 100644 --- a/benchmarks/backend_request_func.py +++ b/benchmarks/backend_request_func.py @@ -620,7 +620,7 @@ def get_tokenizer( kwargs["use_fast"] = False if tokenizer_mode == "mistral": try: - from vllm.transformers_utils.tokenizer import MistralTokenizer + from vllm.transformers_utils.tokenizers import MistralTokenizer except ImportError as e: raise ImportError( "MistralTokenizer requires vllm package.\n" diff --git a/tests/models/language/generation/test_mistral.py b/tests/models/language/generation/test_mistral.py index 80e337d570a3..58efba575f1d 100644 --- a/tests/models/language/generation/test_mistral.py +++ b/tests/models/language/generation/test_mistral.py @@ -10,7 +10,7 @@ MistralToolParser, ) from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizer import MistralTokenizer +from vllm.transformers_utils.tokenizers import MistralTokenizer from ...utils import check_logprobs_close diff --git a/tests/models/multimodal/generation/test_voxtral.py b/tests/models/multimodal/generation/test_voxtral.py index 18a50c3a555d..436f521b2a96 100644 --- a/tests/models/multimodal/generation/test_voxtral.py +++ b/tests/models/multimodal/generation/test_voxtral.py @@ -9,7 +9,7 @@ from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk from mistral_common.protocol.instruct.messages import UserMessage -from vllm.transformers_utils.tokenizer import MistralTokenizer +from vllm.transformers_utils.tokenizers import MistralTokenizer from ....conftest import AudioTestAssets from ....utils import RemoteOpenAIServer diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 4ca99531d27d..2498a03178d1 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -51,7 +51,7 @@ from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path from vllm.transformers_utils.processor import cached_get_processor -from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike +from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike from vllm.utils import random_uuid from vllm.utils.func_utils import supports_kw diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index adf4dfbfd8d6..db9939cba69d 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -62,8 +62,9 @@ from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike from vllm.transformers_utils.tokenizers import ( + MistralTokenizer, + TokenizerLike, maybe_serialize_tool_calls, truncate_tool_call_ids, validate_request_params, diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 9d204bcffcc4..9a89c756a3e8 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -101,7 +101,7 @@ extract_trace_headers, log_tracing_disabled_warning, ) -from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike +from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike from vllm.utils import random_uuid from vllm.utils.async_utils import ( AsyncMicrobatchTokenizer, diff --git a/vllm/entrypoints/openai/serving_score.py b/vllm/entrypoints/openai/serving_score.py index 173815093316..bb253e263895 100644 --- a/vllm/entrypoints/openai/serving_score.py +++ b/vllm/entrypoints/openai/serving_score.py @@ -36,7 +36,7 @@ from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput -from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike +from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike from vllm.utils.async_utils import make_async, merge_async_iterators logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py index 03a807c40f4e..4f7cfe8dd48f 100644 --- a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py @@ -22,7 +22,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike +from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike logger = init_logger(__name__) @@ -31,9 +31,9 @@ class Hermes2ProToolParser(ToolParser): def __init__(self, tokenizer: TokenizerLike): super().__init__(tokenizer) - if isinstance(self.model_tokenizer, MistralTokenizer): + if isinstance(tokenizer, MistralTokenizer): logger.error("Detected Mistral tokenizer when using a Hermes model") - self.model_tokenizer = self.model_tokenizer.tokenizer + self.model_tokenizer = tokenizer.tokenizer self.current_tool_name_sent: bool = False self.prev_tool_call_arr: list[dict] = [] diff --git a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py index 8d7b57423469..620c8dd59322 100644 --- a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py @@ -25,7 +25,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizer import MistralTokenizer, TokenizerLike +from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike logger = init_logger(__name__) From 72afec31ccd975ccd8aba71b6877fb420fade6ac Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:31:57 +0000 Subject: [PATCH 19/31] Move Signed-off-by: DarkLight1337 --- .../test_tokenizer_registry.py | 3 +-- vllm/entrypoints/openai/serving_chat.py | 4 ++-- vllm/transformers_utils/tokenizer.py | 12 +++++------- vllm/transformers_utils/tokenizer_base.py | 10 +++++----- vllm/transformers_utils/tokenizers/__init__.py | 16 +++------------- .../{ => tokenizers}/registry.py | 2 +- 6 files changed, 17 insertions(+), 30 deletions(-) rename vllm/transformers_utils/{ => tokenizers}/registry.py (95%) diff --git a/tests/transformers_utils/test_tokenizer_registry.py b/tests/transformers_utils/test_tokenizer_registry.py index 16b25940111d..d12583fdd555 100644 --- a/tests/transformers_utils/test_tokenizer_registry.py +++ b/tests/transformers_utils/test_tokenizer_registry.py @@ -1,8 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from vllm.transformers_utils.registry import TokenizerRegistry from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.transformers_utils.tokenizers import TokenizerLike, TokenizerRegistry class TestTokenizer(TokenizerLike): diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index db9939cba69d..19b0d16967f1 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -62,9 +62,9 @@ from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.transformers_utils.tokenizers import ( +from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.transformers_utils.tokenizers.mistral import ( MistralTokenizer, - TokenizerLike, maybe_serialize_tool_calls, truncate_tool_call_ids, validate_request_params, diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 8fdd682e021b..bb5c25d1e3b7 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -19,15 +19,13 @@ from .config import get_sentence_transformer_tokenizer_config from .gguf_utils import get_gguf_file_path_from_hf -from .registry import TokenizerRegistry from .repo_utils import list_filtered_repo_files +from .tokenizers import TokenizerLike, TokenizerRegistry from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf if TYPE_CHECKING: from vllm.config import ModelConfig - from .tokenizers import TokenizerLike - logger = init_logger(__name__) @@ -50,7 +48,7 @@ def __getattr__(name: str): def decode_tokens( - tokenizer: "TokenizerLike", + tokenizer: TokenizerLike, token_ids: list[int], *, skip_special_tokens: bool | None = None, @@ -69,7 +67,7 @@ def decode_tokens( def encode_tokens( - tokenizer: "TokenizerLike", + tokenizer: TokenizerLike, text: str, *, truncation: bool | None = None, @@ -97,7 +95,7 @@ def encode_tokens( return tokenizer.encode(text, **kw_args) -def get_cached_tokenizer(tokenizer: "TokenizerLike") -> "TokenizerLike": +def get_cached_tokenizer(tokenizer: TokenizerLike) -> TokenizerLike: """ By default, transformers will recompute multiple tokenizer properties each time they are called, leading to a significant slowdown. @@ -155,7 +153,7 @@ def get_tokenizer( revision: str | None = None, download_dir: str | None = None, **kwargs, -) -> "TokenizerLike": +) -> TokenizerLike: """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" if envs.VLLM_USE_MODELSCOPE: # download model from ModelScope hub, diff --git a/vllm/transformers_utils/tokenizer_base.py b/vllm/transformers_utils/tokenizer_base.py index 5bebadd29609..b658320d9d67 100644 --- a/vllm/transformers_utils/tokenizer_base.py +++ b/vllm/transformers_utils/tokenizer_base.py @@ -9,8 +9,8 @@ def __getattr__(name: str): from .tokenizers import TokenizerLike warnings.warn( - "`vllm.transformers_utils.tokenizer_base.TokenizerBase` " - "has been moved to `vllm.transformers_utils.tokenizers.TokenizerLike`. " + "`vllm.transformers_utils.tokenizer_base.TokenizerBase` has been " + "moved to `vllm.transformers_utils.tokenizers.TokenizerLike`. " "The old name will be removed in v0.13.", DeprecationWarning, stacklevel=2, @@ -18,11 +18,11 @@ def __getattr__(name: str): return TokenizerLike if name == "TokenizerRegistry": - from .registry import TokenizerRegistry + from .tokenizers import TokenizerRegistry warnings.warn( - "`vllm.transformers_utils.tokenizer_base.TokenizerRegistry` " - "has been moved to `vllm.transformers_utils.registry.TokenizerRegistry`. " + "`vllm.transformers_utils.tokenizer_base.TokenizerRegistry` has been " + "moved to `vllm.transformers_utils.tokenizers.TokenizerRegistry`. " "The old name will be removed in v0.13.", DeprecationWarning, stacklevel=2, diff --git a/vllm/transformers_utils/tokenizers/__init__.py b/vllm/transformers_utils/tokenizers/__init__.py index 506ffb5dd597..e26b4e8797ec 100644 --- a/vllm/transformers_utils/tokenizers/__init__.py +++ b/vllm/transformers_utils/tokenizers/__init__.py @@ -1,18 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from .mistral import ( - MistralTokenizer, - maybe_serialize_tool_calls, - truncate_tool_call_ids, - validate_request_params, -) +from .mistral import MistralTokenizer from .protocol import TokenizerLike +from .registry import TokenizerRegistry -__all__ = [ - "TokenizerLike", - "MistralTokenizer", - "maybe_serialize_tool_calls", - "truncate_tool_call_ids", - "validate_request_params", -] +__all__ = ["TokenizerLike", "MistralTokenizer", "TokenizerRegistry"] diff --git a/vllm/transformers_utils/registry.py b/vllm/transformers_utils/tokenizers/registry.py similarity index 95% rename from vllm/transformers_utils/registry.py rename to vllm/transformers_utils/tokenizers/registry.py index 420b61f72b2b..21aa84eb551e 100644 --- a/vllm/transformers_utils/registry.py +++ b/vllm/transformers_utils/tokenizers/registry.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from .tokenizers import TokenizerLike + from .protocol import TokenizerLike class TokenizerRegistry: From d77156a12b46b5bab6518a4a151a32e3d660244d Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:32:25 +0000 Subject: [PATCH 20/31] SImplify Signed-off-by: DarkLight1337 --- vllm/transformers_utils/tokenizers/registry.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vllm/transformers_utils/tokenizers/registry.py b/vllm/transformers_utils/tokenizers/registry.py index 21aa84eb551e..3a236c99b356 100644 --- a/vllm/transformers_utils/tokenizers/registry.py +++ b/vllm/transformers_utils/tokenizers/registry.py @@ -1,10 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib -from typing import TYPE_CHECKING -if TYPE_CHECKING: - from .protocol import TokenizerLike +from .protocol import TokenizerLike class TokenizerRegistry: From 80ac48da5451f239fe82be9e389463ed896af129 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:33:14 +0000 Subject: [PATCH 21/31] Fix Signed-off-by: DarkLight1337 --- tests/multimodal/test_processing.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index 4d2d17424540..b662eeb74717 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import time from contextlib import nullcontext from typing import cast @@ -1088,9 +1089,7 @@ def test_apply_matches_no_match_exits_quickly(): With the fix, it should exit immediately when no match is found. """ - import time - - mock_tokenizer = cast(AnyTokenizer, object()) + mock_tokenizer = cast(TokenizerLike, object()) # Create a long prompt with no placeholder long_prompt = "x" * 10000 From ae6eaac1b26c6eb4287083b47f85c4cdf5ac0055 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:40:30 +0000 Subject: [PATCH 22/31] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/score_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/score_utils.py b/vllm/entrypoints/score_utils.py index afc7bf8c3354..04d5a192918d 100644 --- a/vllm/entrypoints/score_utils.py +++ b/vllm/entrypoints/score_utils.py @@ -52,7 +52,7 @@ def _cosine_similarity( pair_score = scorer(emb_1.outputs.data, emb_2.outputs.data) padding = [] - if (pad_token_id := tokenizer.pad_token_id) is not None: + if (pad_token_id := getattr(tokenizer, "pad_token_id", None)) is not None: padding = [pad_token_id] tokens = emb_1.prompt_token_ids + padding + emb_2.prompt_token_ids From 7b6feb2a433c8188407a0f3caca1f7e12bb21b73 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 00:41:49 +0000 Subject: [PATCH 23/31] Update error message Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_completion.py | 3 +-- vllm/entrypoints/openai/serving_engine.py | 4 ++-- vllm/v1/engine/async_llm.py | 2 +- vllm/v1/engine/llm_engine.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index e96d92428e46..fa9cf73d98e7 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -647,8 +647,7 @@ def _create_completion_logprobs( else: if tokenizer is None: raise ValueError( - "Unable to get tokenizer because " - "skip_tokenizer_init is True" + "Unable to get tokenizer because `skip_tokenizer_init=True`" ) token = tokenizer.decode(token_id) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 9a89c756a3e8..7aab13831d4c 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1097,7 +1097,7 @@ async def _preprocess_chat( ]: if tokenizer is None: raise ValueError( - "Unable to get tokenizer because skip_tokenizer_init is True" + "Unable to get tokenizer because `skip_tokenizer_init=True`" ) model_config = self.model_config @@ -1458,7 +1458,7 @@ def _get_decoded_token( if tokenizer is None: raise ValueError( - "Unable to get tokenizer because skip_tokenizer_init is True" + "Unable to get tokenizer because `skip_tokenizer_init=True`" ) return tokenizer.decode(token_id) diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 77d4d0496748..487a811f9af1 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -715,7 +715,7 @@ def tokenizer(self, tokenizer: TokenizerLike | None) -> None: async def get_tokenizer(self) -> TokenizerLike: if self.tokenizer is None: raise ValueError( - "Unable to get tokenizer because skip_tokenizer_init is True" + "Unable to get tokenizer because `skip_tokenizer_init=True`" ) return self.tokenizer diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 89d02e8e14d8..9812197cec2c 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -362,7 +362,7 @@ def tokenizer(self, tokenizer: TokenizerLike | None) -> None: def get_tokenizer(self) -> TokenizerLike: if self.tokenizer is None: raise ValueError( - "Unable to get tokenizer because skip_tokenizer_init is True" + "Unable to get tokenizer because `skip_tokenizer_init=True`" ) return self.tokenizer From 3866bae8fd05d2532840fc02873b9015be6a5f47 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 04:59:22 +0000 Subject: [PATCH 24/31] Move Signed-off-by: DarkLight1337 --- .buildkite/test-amd.yaml | 7 +++- .buildkite/test-pipeline.yaml | 7 +++- benchmarks/backend_request_func.py | 2 +- .../entrypoints/openai/test_serving_engine.py | 2 +- .../openai/tool_parsers/conftest.py | 2 +- .../tool_parsers/test_hermes_tool_parser.py | 2 +- .../test_llama3_json_tool_parser.py | 2 +- .../test_llama4_pythonic_tool_parser.py | 2 +- .../tool_parsers/test_olmo3_tool_parser.py | 2 +- .../tool_parsers/test_pythonic_tool_parser.py | 2 +- .../entrypoints/openai/tool_parsers/utils.py | 2 +- tests/entrypoints/test_chat_utils.py | 2 +- .../language/generation/test_mistral.py | 2 +- .../multimodal/generation/test_voxtral.py | 2 +- .../multimodal/generation/vlm_utils/core.py | 2 +- .../multimodal/generation/vlm_utils/types.py | 2 +- .../multimodal/processing/test_common.py | 2 +- tests/multimodal/test_processing.py | 2 +- .../test_mistral_reasoning_parser.py | 2 +- tests/reasoning/utils.py | 2 +- tests/tokenizers/__init__.py | 0 .../test_basic.py} | 40 +++++++++++-------- .../test_cached_tokenizer.py | 2 +- .../test_detokenize.py | 2 +- .../test_mistral.py} | 2 +- .../test_registry.py} | 4 +- .../tool_use/test_ernie45_moe_tool_parser.py | 2 +- tests/tool_use/test_jamba_tool_parser.py | 2 +- tests/tool_use/test_qwen3coder_tool_parser.py | 2 +- tests/tool_use/test_seed_oss_tool_parser.py | 2 +- tests/tool_use/test_xlam_tool_parser.py | 2 +- tests/v1/engine/test_output_processor.py | 2 +- tools/pre_commit/check_pickle_imports.py | 2 +- tools/pre_commit/mypy.py | 1 + vllm/benchmarks/datasets.py | 2 +- vllm/engine/protocol.py | 2 +- vllm/entrypoints/chat_utils.py | 2 +- vllm/entrypoints/llm.py | 2 +- vllm/entrypoints/openai/serving_chat.py | 4 +- vllm/entrypoints/openai/serving_completion.py | 2 +- vllm/entrypoints/openai/serving_engine.py | 2 +- vllm/entrypoints/openai/serving_responses.py | 2 +- vllm/entrypoints/openai/serving_score.py | 2 +- .../openai/serving_tokenization.py | 2 +- .../tool_parsers/abstract_tool_parser.py | 2 +- .../tool_parsers/deepseekv31_tool_parser.py | 2 +- .../tool_parsers/deepseekv3_tool_parser.py | 2 +- .../tool_parsers/ernie45_tool_parser.py | 2 +- .../tool_parsers/glm4_moe_tool_parser.py | 2 +- .../granite_20b_fc_tool_parser.py | 2 +- .../tool_parsers/granite_tool_parser.py | 2 +- .../openai/tool_parsers/hermes_tool_parser.py | 2 +- .../tool_parsers/hunyuan_a13b_tool_parser.py | 2 +- .../tool_parsers/internlm2_tool_parser.py | 2 +- .../openai/tool_parsers/jamba_tool_parser.py | 2 +- .../tool_parsers/kimi_k2_tool_parser.py | 2 +- .../tool_parsers/longcat_tool_parser.py | 2 +- .../tool_parsers/minimax_m2_tool_parser.py | 2 +- .../tool_parsers/minimax_tool_parser.py | 2 +- .../tool_parsers/mistral_tool_parser.py | 2 +- .../openai/tool_parsers/openai_tool_parser.py | 2 +- .../tool_parsers/qwen3coder_tool_parser.py | 2 +- .../tool_parsers/qwen3xml_tool_parser.py | 2 +- .../tool_parsers/seed_oss_tool_parser.py | 2 +- .../openai/tool_parsers/step3_tool_parser.py | 2 +- .../openai/tool_parsers/xlam_tool_parser.py | 2 +- vllm/entrypoints/renderer.py | 2 +- vllm/entrypoints/utils.py | 2 +- vllm/inputs/preprocess.py | 2 +- vllm/logits_process.py | 2 +- vllm/model_executor/models/h2ovl.py | 2 +- vllm/model_executor/models/internvl.py | 2 +- .../model_executor/models/nano_nemotron_vl.py | 2 +- vllm/model_executor/models/nemotron_vl.py | 2 +- vllm/model_executor/models/opencua.py | 2 +- vllm/model_executor/models/pixtral.py | 2 +- vllm/model_executor/models/qwen2_vl.py | 2 +- vllm/model_executor/models/skyworkr1v.py | 2 +- vllm/model_executor/models/step3_vl.py | 2 +- vllm/model_executor/models/voxtral.py | 2 +- vllm/multimodal/processing.py | 2 +- vllm/multimodal/registry.py | 2 +- vllm/reasoning/abs_reasoning_parsers.py | 2 +- vllm/reasoning/basic_parsers.py | 2 +- vllm/reasoning/minimax_m2_reasoning_parser.py | 2 +- vllm/reasoning/mistral_reasoning_parser.py | 2 +- vllm/reasoning/olmo3_reasoning_parser.py | 2 +- vllm/sampling_params.py | 2 +- .../tokenizers/__init__.py | 0 .../tokenizers/mistral.py | 0 .../tokenizers/protocol.py | 5 --- .../tokenizers/registry.py | 0 vllm/transformers_utils/detokenizer_utils.py | 2 +- vllm/transformers_utils/tokenizer.py | 8 ++-- vllm/transformers_utils/tokenizer_base.py | 8 ++-- vllm/v1/engine/async_llm.py | 2 +- vllm/v1/engine/input_processor.py | 3 +- vllm/v1/engine/llm_engine.py | 2 +- vllm/v1/engine/output_processor.py | 2 +- vllm/v1/structured_output/backend_types.py | 2 +- vllm/v1/structured_output/backend_xgrammar.py | 2 +- vllm/v1/structured_output/utils.py | 2 +- 102 files changed, 134 insertions(+), 129 deletions(-) create mode 100644 tests/tokenizers/__init__.py rename tests/{transformers_utils/test_tokenizer.py => tokenizers/test_basic.py} (65%) rename tests/{transformers_utils => tokenizers}/test_cached_tokenizer.py (96%) rename tests/{transformers_utils => tokenizers}/test_detokenize.py (99%) rename tests/{transformers_utils/test_mistral_tokenizer.py => tokenizers/test_mistral.py} (99%) rename tests/{transformers_utils/test_tokenizer_registry.py => tokenizers/test_registry.py} (87%) rename vllm/{transformers_utils => }/tokenizers/__init__.py (100%) rename vllm/{transformers_utils => }/tokenizers/mistral.py (100%) rename vllm/{transformers_utils => }/tokenizers/protocol.py (97%) rename vllm/{transformers_utils => }/tokenizers/registry.py (100%) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index f41ce1eb6942..30e5dd9a3453 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -61,8 +61,8 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 10min - timeout_in_minutes: 15 +- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 4 mins + timeout_in_minutes: 10 mirror_hardwares: [amdexperimental, amdproduction] agent_pool: mi325_1 # grade: Blocking @@ -316,12 +316,15 @@ steps: source_file_dependencies: - vllm/ - tests/engine + - tests/tokenizers - tests/test_sequence - tests/test_config - tests/test_logger - tests/test_vllm_port commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py + # OOM in the CI unless we run this separately + - pytest -v -s tokenizers - label: V1 Test e2e + engine # 30min timeout_in_minutes: 45 diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index f55747e1f7ca..ad6e165dca81 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -57,8 +57,8 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 10min - timeout_in_minutes: 15 +- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 4 mins + timeout_in_minutes: 10 source_file_dependencies: - vllm/ - tests/test_inputs.py @@ -282,12 +282,15 @@ steps: source_file_dependencies: - vllm/ - tests/engine + - tests/tokenizers - tests/test_sequence - tests/test_config - tests/test_logger - tests/test_vllm_port commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py + # OOM in the CI unless we run this separately + - pytest -v -s tokenizers - label: V1 Test e2e + engine # 30min timeout_in_minutes: 45 diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py index f08ac66b9bc5..d69d74ca61f5 100644 --- a/benchmarks/backend_request_func.py +++ b/benchmarks/backend_request_func.py @@ -620,7 +620,7 @@ def get_tokenizer( kwargs["use_fast"] = False if tokenizer_mode == "mistral": try: - from vllm.transformers_utils.tokenizers import MistralTokenizer + from vllm.tokenizers import MistralTokenizer except ImportError as e: raise ImportError( "MistralTokenizer requires vllm package.\n" diff --git a/tests/entrypoints/openai/test_serving_engine.py b/tests/entrypoints/openai/test_serving_engine.py index 29892d0bf38a..956a06dc5487 100644 --- a/tests/entrypoints/openai/test_serving_engine.py +++ b/tests/entrypoints/openai/test_serving_engine.py @@ -10,7 +10,7 @@ from vllm.config import ModelConfig from vllm.entrypoints.openai.serving_engine import OpenAIServing from vllm.entrypoints.openai.serving_models import OpenAIServingModels -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer +from vllm.tokenizers import MistralTokenizer @pytest.fixture() diff --git a/tests/entrypoints/openai/tool_parsers/conftest.py b/tests/entrypoints/openai/tool_parsers/conftest.py index b748878a377d..a40d0ab44cf7 100644 --- a/tests/entrypoints/openai/tool_parsers/conftest.py +++ b/tests/entrypoints/openai/tool_parsers/conftest.py @@ -4,7 +4,7 @@ import pytest from transformers import AutoTokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike @pytest.fixture(scope="function") diff --git a/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py index da4f35383aa2..b2303ab0e7b7 100644 --- a/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py @@ -7,7 +7,7 @@ from vllm.entrypoints.openai.protocol import ChatCompletionRequest from vllm.entrypoints.openai.tool_parsers.hermes_tool_parser import Hermes2ProToolParser -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from ....utils import RemoteOpenAIServer diff --git a/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py index 4d519f1943b6..6c286ca90ce4 100644 --- a/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_llama3_json_tool_parser.py @@ -7,7 +7,7 @@ from vllm.entrypoints.openai.protocol import ExtractedToolCallInformation from vllm.entrypoints.openai.tool_parsers.llama_tool_parser import Llama3JsonToolParser -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike @pytest.fixture diff --git a/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py index a7c1856ca8c5..8aa88a007188 100644 --- a/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_llama4_pythonic_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike # Test cases similar to pythonic parser but with Llama4 specific format SIMPLE_FUNCTION_OUTPUT = "[get_weather(city='LA', metric='C')]" diff --git a/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py index 92fc68fe30ce..a0b9a3c563bc 100644 --- a/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_olmo3_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')" diff --git a/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py b/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py index fb3c8a62e21e..52202c55e840 100644 --- a/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py +++ b/tests/entrypoints/openai/tool_parsers/test_pythonic_tool_parser.py @@ -11,7 +11,7 @@ ) from vllm.entrypoints.openai.protocol import FunctionCall from vllm.entrypoints.openai.tool_parsers import ToolParser, ToolParserManager -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike # https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/text_prompt_format.md#model-response-format-1 SIMPLE_FUNCTION_OUTPUT = "get_weather(city='San Francisco', metric='celsius')" diff --git a/tests/entrypoints/openai/tool_parsers/utils.py b/tests/entrypoints/openai/tool_parsers/utils.py index 3e7c39664454..2d4f5f173410 100644 --- a/tests/entrypoints/openai/tool_parsers/utils.py +++ b/tests/entrypoints/openai/tool_parsers/utils.py @@ -11,7 +11,7 @@ ToolCall, ) from vllm.entrypoints.openai.tool_parsers import ToolParser -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike class StreamingToolReconstructor: diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index 7baf564ad01a..a351cda60621 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -28,8 +28,8 @@ encode_image_base64, encode_video_base64, ) +from vllm.tokenizers import MistralTokenizer from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer from ..models.registry import HF_EXAMPLE_MODELS from ..utils import VLLM_PATH diff --git a/tests/models/language/generation/test_mistral.py b/tests/models/language/generation/test_mistral.py index 58efba575f1d..1377776a6d84 100644 --- a/tests/models/language/generation/test_mistral.py +++ b/tests/models/language/generation/test_mistral.py @@ -10,7 +10,7 @@ MistralToolParser, ) from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizers import MistralTokenizer +from vllm.tokenizers import MistralTokenizer from ...utils import check_logprobs_close diff --git a/tests/models/multimodal/generation/test_voxtral.py b/tests/models/multimodal/generation/test_voxtral.py index 436f521b2a96..9e9087cb0fc4 100644 --- a/tests/models/multimodal/generation/test_voxtral.py +++ b/tests/models/multimodal/generation/test_voxtral.py @@ -9,7 +9,7 @@ from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk from mistral_common.protocol.instruct.messages import UserMessage -from vllm.transformers_utils.tokenizers import MistralTokenizer +from vllm.tokenizers import MistralTokenizer from ....conftest import AudioTestAssets from ....utils import RemoteOpenAIServer diff --git a/tests/models/multimodal/generation/vlm_utils/core.py b/tests/models/multimodal/generation/vlm_utils/core.py index 5ea1c3ed5d99..08cf4b2202dc 100644 --- a/tests/models/multimodal/generation/vlm_utils/core.py +++ b/tests/models/multimodal/generation/vlm_utils/core.py @@ -9,7 +9,7 @@ from transformers.models.auto.auto_factory import _BaseAutoModelClass from vllm.config.model import RunnerOption -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from .....conftest import HfRunner, VllmRunner from ....registry import HF_EXAMPLE_MODELS diff --git a/tests/models/multimodal/generation/vlm_utils/types.py b/tests/models/multimodal/generation/vlm_utils/types.py index 261c26fff2d4..0c03c8449712 100644 --- a/tests/models/multimodal/generation/vlm_utils/types.py +++ b/tests/models/multimodal/generation/vlm_utils/types.py @@ -14,7 +14,7 @@ from vllm.config.model import RunnerOption from vllm.logprobs import SampleLogprobs -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from .....conftest import ( AUDIO_ASSETS, diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 4041f5af9ceb..c39e52210090 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -22,11 +22,11 @@ from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.inputs import MultiModalInputs from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext +from vllm.tokenizers import MistralTokenizer from vllm.transformers_utils.tokenizer import ( cached_tokenizer_from_config, encode_tokens, ) -from vllm.transformers_utils.tokenizers import MistralTokenizer from ....multimodal.utils import random_audio, random_image, random_video from ...registry import ( diff --git a/tests/multimodal/test_processing.py b/tests/multimodal/test_processing.py index b662eeb74717..f7fa8da54d54 100644 --- a/tests/multimodal/test_processing.py +++ b/tests/multimodal/test_processing.py @@ -24,7 +24,7 @@ replace_token_matches, ) from vllm.multimodal.profiling import MultiModalProfiler -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from .utils import random_image diff --git a/tests/reasoning/test_mistral_reasoning_parser.py b/tests/reasoning/test_mistral_reasoning_parser.py index 5163c863863a..0fe315c2567f 100644 --- a/tests/reasoning/test_mistral_reasoning_parser.py +++ b/tests/reasoning/test_mistral_reasoning_parser.py @@ -5,7 +5,7 @@ from tests.reasoning.utils import run_reasoning_extraction_mistral from vllm.reasoning import ReasoningParser, ReasoningParserManager -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer +from vllm.tokenizers import MistralTokenizer parser_name = "mistral" diff --git a/tests/reasoning/utils.py b/tests/reasoning/utils.py index bd0b230a847c..695312a0cadf 100644 --- a/tests/reasoning/utils.py +++ b/tests/reasoning/utils.py @@ -4,7 +4,7 @@ from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage from vllm.reasoning import ReasoningParser -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer +from vllm.tokenizers import MistralTokenizer class StreamingReasoningReconstructor: diff --git a/tests/tokenizers/__init__.py b/tests/tokenizers/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/transformers_utils/test_tokenizer.py b/tests/tokenizers/test_basic.py similarity index 65% rename from tests/transformers_utils/test_tokenizer.py rename to tests/tokenizers/test_basic.py index ac0260913422..1fca633cc5cd 100644 --- a/tests/transformers_utils/test_tokenizer.py +++ b/tests/tokenizers/test_basic.py @@ -5,8 +5,8 @@ import pytest from transformers import PreTrainedTokenizerBase +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike def _get_missing_attrs(obj: object, target: type): @@ -14,22 +14,28 @@ def _get_missing_attrs(obj: object, target: type): def test_tokenizer_like_protocol(): - assert isinstance( - tokenizer := get_tokenizer("gpt2", use_fast=False), - TokenizerLike, - ), f"Missing attrs: {_get_missing_attrs(tokenizer, TokenizerLike)}" - - assert isinstance( - tokenizer := get_tokenizer("gpt2", use_fast=True), - TokenizerLike, - ), f"Missing attrs: {_get_missing_attrs(tokenizer, TokenizerLike)}" - - assert isinstance( - tokenizer := get_tokenizer( - "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral" - ), - TokenizerLike, - ), f"Missing attrs: {_get_missing_attrs(tokenizer, TokenizerLike)}" + assert not ( + missing_attrs := _get_missing_attrs( + get_tokenizer("gpt2", use_fast=False), + TokenizerLike, + ) + ), f"Missing attrs: {missing_attrs}" + + assert not ( + missing_attrs := _get_missing_attrs( + get_tokenizer("gpt2", use_fast=True), + TokenizerLike, + ) + ), f"Missing attrs: {missing_attrs}" + + assert not ( + missing_attrs := _get_missing_attrs( + get_tokenizer( + "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral" + ), + TokenizerLike, + ) + ), f"Missing attrs: {missing_attrs}" @pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"]) diff --git a/tests/transformers_utils/test_cached_tokenizer.py b/tests/tokenizers/test_cached_tokenizer.py similarity index 96% rename from tests/transformers_utils/test_cached_tokenizer.py rename to tests/tokenizers/test_cached_tokenizer.py index 638feaa2bc33..48234687ea1e 100644 --- a/tests/transformers_utils/test_cached_tokenizer.py +++ b/tests/tokenizers/test_cached_tokenizer.py @@ -6,8 +6,8 @@ import pytest from transformers import AutoTokenizer +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.tokenizer import get_cached_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike @pytest.mark.parametrize("model_id", ["gpt2", "zai-org/chatglm3-6b"]) diff --git a/tests/transformers_utils/test_detokenize.py b/tests/tokenizers/test_detokenize.py similarity index 99% rename from tests/transformers_utils/test_detokenize.py rename to tests/tokenizers/test_detokenize.py index f4b43a21daaa..ae1d6b095672 100644 --- a/tests/transformers_utils/test_detokenize.py +++ b/tests/tokenizers/test_detokenize.py @@ -8,7 +8,7 @@ from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer +from vllm.tokenizers import MistralTokenizer from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.detokenizer import ( FastIncrementalDetokenizer, diff --git a/tests/transformers_utils/test_mistral_tokenizer.py b/tests/tokenizers/test_mistral.py similarity index 99% rename from tests/transformers_utils/test_mistral_tokenizer.py rename to tests/tokenizers/test_mistral.py index 2b991f01d7a2..0706a94791dc 100644 --- a/tests/transformers_utils/test_mistral_tokenizer.py +++ b/tests/tokenizers/test_mistral.py @@ -7,7 +7,7 @@ from mistral_common.exceptions import InvalidMessageStructureException from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy -from vllm.transformers_utils.tokenizers.mistral import ( +from vllm.tokenizers.mistral import ( MistralTokenizer, _prepare_apply_chat_template_tools_and_messages, ) diff --git a/tests/transformers_utils/test_tokenizer_registry.py b/tests/tokenizers/test_registry.py similarity index 87% rename from tests/transformers_utils/test_tokenizer_registry.py rename to tests/tokenizers/test_registry.py index d12583fdd555..ad085eedefdb 100644 --- a/tests/transformers_utils/test_tokenizer_registry.py +++ b/tests/tokenizers/test_registry.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from vllm.tokenizers import TokenizerLike, TokenizerRegistry from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike, TokenizerRegistry class TestTokenizer(TokenizerLike): @@ -21,7 +21,7 @@ def eos_token_id(self) -> int: def test_customized_tokenizer(): TokenizerRegistry.register( "test_tokenizer", - "tests.transformers_utils.test_tokenizer_registry", + "tests.tokenizers.test_registry", "TestTokenizer", ) diff --git a/tests/tool_use/test_ernie45_moe_tool_parser.py b/tests/tool_use/test_ernie45_moe_tool_parser.py index c7a090888a5f..ee9da4fd6464 100644 --- a/tests/tool_use/test_ernie45_moe_tool_parser.py +++ b/tests/tool_use/test_ernie45_moe_tool_parser.py @@ -14,9 +14,9 @@ ToolCall, ) from vllm.entrypoints.openai.tool_parsers.ernie45_tool_parser import Ernie45ToolParser +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike # Use a common model that is likely to be available MODEL = "baidu/ERNIE-4.5-21B-A3B-Thinking" diff --git a/tests/tool_use/test_jamba_tool_parser.py b/tests/tool_use/test_jamba_tool_parser.py index 572d5476b2d5..2413b983fe87 100644 --- a/tests/tool_use/test_jamba_tool_parser.py +++ b/tests/tool_use/test_jamba_tool_parser.py @@ -10,9 +10,9 @@ from vllm.entrypoints.openai.protocol import DeltaMessage, FunctionCall, ToolCall from vllm.entrypoints.openai.tool_parsers.jamba_tool_parser import JambaToolParser +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike pytestmark = pytest.mark.cpu_test diff --git a/tests/tool_use/test_qwen3coder_tool_parser.py b/tests/tool_use/test_qwen3coder_tool_parser.py index b3a65e0636d0..3cf1f4ef89f1 100644 --- a/tests/tool_use/test_qwen3coder_tool_parser.py +++ b/tests/tool_use/test_qwen3coder_tool_parser.py @@ -17,9 +17,9 @@ Qwen3CoderToolParser, ) from vllm.entrypoints.openai.tool_parsers.qwen3xml_tool_parser import Qwen3XMLToolParser +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike pytestmark = pytest.mark.cpu_test diff --git a/tests/tool_use/test_seed_oss_tool_parser.py b/tests/tool_use/test_seed_oss_tool_parser.py index 855b9ec104c1..8e1ad5e9cedc 100644 --- a/tests/tool_use/test_seed_oss_tool_parser.py +++ b/tests/tool_use/test_seed_oss_tool_parser.py @@ -15,9 +15,9 @@ ToolCall, ) from vllm.entrypoints.openai.tool_parsers.seed_oss_tool_parser import SeedOssToolParser +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike pytestmark = pytest.mark.cpu_test diff --git a/tests/tool_use/test_xlam_tool_parser.py b/tests/tool_use/test_xlam_tool_parser.py index 9799f34af28a..a1852c368eeb 100644 --- a/tests/tool_use/test_xlam_tool_parser.py +++ b/tests/tool_use/test_xlam_tool_parser.py @@ -13,9 +13,9 @@ ToolCall, ) from vllm.entrypoints.openai.tool_parsers.xlam_tool_parser import xLAMToolParser +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.detokenizer_utils import detokenize_incrementally from vllm.transformers_utils.tokenizer import get_tokenizer -from vllm.transformers_utils.tokenizers import TokenizerLike pytestmark = pytest.mark.cpu_test diff --git a/tests/v1/engine/test_output_processor.py b/tests/v1/engine/test_output_processor.py index 13e59c58b45b..990aa9d92585 100644 --- a/tests/v1/engine/test_output_processor.py +++ b/tests/v1/engine/test_output_processor.py @@ -18,7 +18,7 @@ from vllm.lora.request import LoRARequest from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import RequestOutputKind, SamplingParams -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.v1.engine import ( EngineCoreEvent, EngineCoreEventType, diff --git a/tools/pre_commit/check_pickle_imports.py b/tools/pre_commit/check_pickle_imports.py index 0a12fe818670..b738d8437cc7 100644 --- a/tools/pre_commit/check_pickle_imports.py +++ b/tools/pre_commit/check_pickle_imports.py @@ -27,8 +27,8 @@ "vllm/distributed/device_communicators/shm_broadcast.py", "vllm/distributed/device_communicators/shm_object_storage.py", "vllm/utils/hashing.py", + "tests/tokenizers/test_cached_tokenizer.py", "tests/utils_/test_hashing.py", - "tests/transformers_utils/test_cached_tokenizer.py", "benchmarks/kernels/graph_machete_bench.py", "benchmarks/kernels/benchmark_lora.py", "benchmarks/kernels/benchmark_machete.py", diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 47e01fc93b48..724b39304426 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -35,6 +35,7 @@ "vllm/multimodal", "vllm/platforms", "vllm/plugins", + "vllm/tokenizers", "vllm/transformers_utils", "vllm/triton_utils", "vllm/usage", diff --git a/vllm/benchmarks/datasets.py b/vllm/benchmarks/datasets.py index 2b5cd6e2f89f..ec9b0fd6e969 100644 --- a/vllm/benchmarks/datasets.py +++ b/vllm/benchmarks/datasets.py @@ -39,7 +39,7 @@ from vllm.lora.utils import get_adapter_absolute_path from vllm.multimodal import MultiModalDataDict from vllm.multimodal.image import convert_image_mode -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils.import_utils import PlaceholderModule try: diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 6ff8f9392642..f2b19c845018 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -13,7 +13,7 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.input_processor import InputProcessor diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 2498a03178d1..1643906894c6 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -49,9 +49,9 @@ from vllm.model_executor.models import SupportsMultiModal from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector +from vllm.tokenizers import MistralTokenizer, TokenizerLike from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path from vllm.transformers_utils.processor import cached_get_processor -from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike from vllm.utils import random_uuid from vllm.utils.func_utils import supports_kw diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 37060d30b717..4ea213752e39 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -71,8 +71,8 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams from vllm.tasks import PoolingTask +from vllm.tokenizers import MistralTokenizer, TokenizerLike from vllm.transformers_utils.tokenizer import get_cached_tokenizer -from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike from vllm.usage.usage_lib import UsageContext from vllm.utils.collection_utils import as_iter, is_list_of from vllm.utils.counter import Counter diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 19b0d16967f1..cecd1da1e554 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -62,8 +62,8 @@ from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.transformers_utils.tokenizers import TokenizerLike -from vllm.transformers_utils.tokenizers.mistral import ( +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.mistral import ( MistralTokenizer, maybe_serialize_tool_calls, truncate_tool_call_ids, diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index fa9cf73d98e7..3e421e21e3e8 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -33,7 +33,7 @@ from vllm.logprobs import Logprob from vllm.outputs import RequestOutput from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import merge_async_iterators from vllm.utils.collection_utils import as_list from vllm.v1.sample.logits_processor import validate_logits_processors_parameters diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 7aab13831d4c..48b2248b4e9d 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -96,12 +96,12 @@ from vllm.pooling_params import PoolingParams from vllm.reasoning import ReasoningParser, ReasoningParserManager from vllm.sampling_params import BeamSearchParams, SamplingParams +from vllm.tokenizers import MistralTokenizer, TokenizerLike from vllm.tracing import ( contains_trace_headers, extract_trace_headers, log_tracing_disabled_warning, ) -from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike from vllm.utils import random_uuid from vllm.utils.async_utils import ( AsyncMicrobatchTokenizer, diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index 578e34bacfec..5144916ba71e 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -105,7 +105,7 @@ from vllm.logprobs import SampleLogprobs from vllm.outputs import CompletionOutput from vllm.sampling_params import SamplingParams, StructuredOutputsParams -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/serving_score.py b/vllm/entrypoints/openai/serving_score.py index bb253e263895..0874c01c1f2a 100644 --- a/vllm/entrypoints/openai/serving_score.py +++ b/vllm/entrypoints/openai/serving_score.py @@ -36,7 +36,7 @@ from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput -from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import MistralTokenizer, TokenizerLike from vllm.utils.async_utils import make_async, merge_async_iterators logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/serving_tokenization.py b/vllm/entrypoints/openai/serving_tokenization.py index ec7c7a3e800a..979da02d1450 100644 --- a/vllm/entrypoints/openai/serving_tokenization.py +++ b/vllm/entrypoints/openai/serving_tokenization.py @@ -22,7 +22,7 @@ from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.renderer import RenderConfig from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py index b69599ad3613..87ef2e0786a9 100644 --- a/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/abstract_tool_parser.py @@ -22,7 +22,7 @@ from vllm.sampling_params import ( StructuredOutputsParams, ) -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils.collection_utils import is_list_of from vllm.utils.import_utils import import_from_path diff --git a/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py index 37a9c8b4a735..10de3dabf985 100644 --- a/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/deepseekv31_tool_parser.py @@ -19,7 +19,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py index fa9b1d7f9d1c..66b14875dce2 100644 --- a/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/deepseekv3_tool_parser.py @@ -19,7 +19,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py index aa301e7efaba..d054d8e4b865 100644 --- a/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/ernie45_tool_parser.py @@ -19,7 +19,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py index bdadd737bd7e..165346adb3d9 100644 --- a/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/glm4_moe_tool_parser.py @@ -22,7 +22,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py index 0d872d058d94..df1b590526b1 100644 --- a/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/granite_20b_fc_tool_parser.py @@ -29,7 +29,7 @@ partial_json_loads, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py index 335352d1d6a0..14b0ca0abe35 100644 --- a/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/granite_tool_parser.py @@ -27,7 +27,7 @@ partial_json_loads, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py index 4f7cfe8dd48f..19c1c83268ed 100644 --- a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py @@ -22,7 +22,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import MistralTokenizer, TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py index 37176f55a81f..d2419b5d84ea 100644 --- a/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/hunyuan_a13b_tool_parser.py @@ -22,7 +22,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import consume_space from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py index 72fa8aab736a..67788358543e 100644 --- a/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/internlm2_tool_parser.py @@ -22,7 +22,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py index 9fd3edf5dd6f..4655da8dd454 100644 --- a/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py @@ -21,7 +21,7 @@ from vllm.entrypoints.openai.tool_parsers import ToolParser from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import MistralTokenizer, TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py index f7fe1168b9f7..07db52ebd5af 100644 --- a/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/kimi_k2_tool_parser.py @@ -19,7 +19,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py index 4ed053a56793..76d76a4aa35a 100644 --- a/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/longcat_tool_parser.py @@ -4,7 +4,7 @@ import regex as re from vllm.entrypoints.openai.tool_parsers.hermes_tool_parser import Hermes2ProToolParser -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike class LongcatFlashToolParser(Hermes2ProToolParser): diff --git a/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py index 3062eae87ae9..b595a98f3555 100644 --- a/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/minimax_m2_tool_parser.py @@ -21,7 +21,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py index bb803c0beef8..1025041037c6 100644 --- a/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/minimax_tool_parser.py @@ -22,7 +22,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py index 620c8dd59322..7e2d67a1fb65 100644 --- a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py @@ -25,7 +25,7 @@ ) from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import MistralTokenizer, TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py index 2e927f389250..8bdf35d40880 100644 --- a/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/openai_tool_parser.py @@ -18,7 +18,7 @@ from vllm.logger import init_logger if TYPE_CHECKING: - from vllm.transformers_utils.tokenizers import TokenizerLike + from vllm.tokenizers import TokenizerLike else: TokenizerLike = object diff --git a/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py index 6f1478023edd..d49b14690ef0 100644 --- a/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/qwen3coder_tool_parser.py @@ -22,7 +22,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py index dc12ebc8daec..03862ff432a5 100644 --- a/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/qwen3xml_tool_parser.py @@ -23,7 +23,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py index 7e3bfee3bc5f..c7947faad192 100644 --- a/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/seed_oss_tool_parser.py @@ -25,7 +25,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py index 439327a09219..9213d6859dd9 100644 --- a/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/step3_tool_parser.py @@ -21,7 +21,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py index 7bd4541cf155..effd2bd08b42 100644 --- a/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/xlam_tool_parser.py @@ -21,7 +21,7 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid logger = init_logger(__name__) diff --git a/vllm/entrypoints/renderer.py b/vllm/entrypoints/renderer.py index 25caeae2d8ff..10b90bbbb0f3 100644 --- a/vllm/entrypoints/renderer.py +++ b/vllm/entrypoints/renderer.py @@ -16,7 +16,7 @@ from vllm.inputs.data import TextPrompt as EngineTextPrompt from vllm.inputs.data import TokensPrompt as EngineTokensPrompt from vllm.inputs.parse import get_prompt_components, parse_raw_prompts -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import AsyncMicrobatchTokenizer diff --git a/vllm/entrypoints/utils.py b/vllm/entrypoints/utils.py index 088bb679fef4..daeeb995bc74 100644 --- a/vllm/entrypoints/utils.py +++ b/vllm/entrypoints/utils.py @@ -30,7 +30,7 @@ from vllm.entrypoints.openai.serving_models import LoRAModulePath from vllm.logger import init_logger from vllm.platforms import current_platform -from vllm.transformers_utils.tokenizers import MistralTokenizer +from vllm.tokenizers import MistralTokenizer from vllm.utils.argparse_utils import FlexibleArgumentParser logger = init_logger(__name__) diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py index b803a6f7f6ec..46d1bed38aa8 100644 --- a/vllm/inputs/preprocess.py +++ b/vllm/inputs/preprocess.py @@ -17,7 +17,7 @@ MultiModalUUIDDict, ) from vllm.multimodal.processing import BaseMultiModalProcessor -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils.jsontree import json_iter_leaves from vllm.v1.metrics.stats import MultiModalCacheStats diff --git a/vllm/logits_process.py b/vllm/logits_process.py index d167a899c28e..1bf97c2535fb 100644 --- a/vllm/logits_process.py +++ b/vllm/logits_process.py @@ -5,7 +5,7 @@ import torch -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike LogitsProcessor: TypeAlias = ( Callable[[list[int], torch.Tensor], torch.Tensor] diff --git a/vllm/model_executor/models/h2ovl.py b/vllm/model_executor/models/h2ovl.py index 651b3143f337..6276c3d67541 100644 --- a/vllm/model_executor/models/h2ovl.py +++ b/vllm/model_executor/models/h2ovl.py @@ -28,7 +28,7 @@ PromptUpdate, PromptUpdateDetails, ) -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from .intern_vit import InternVisionModel from .internvl import ( diff --git a/vllm/model_executor/models/internvl.py b/vllm/model_executor/models/internvl.py index 8f52b58838e0..fccddf3a6b29 100644 --- a/vllm/model_executor/models/internvl.py +++ b/vllm/model_executor/models/internvl.py @@ -50,7 +50,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.torch_utils import set_default_torch_num_threads diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 6ce7927b4568..11beeddabe30 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -73,12 +73,12 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.configs.radio import RadioConfig from vllm.transformers_utils.tokenizer import ( cached_tokenizer_from_config, encode_tokens, ) -from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .utils import _merge_multimodal_embeddings diff --git a/vllm/model_executor/models/nemotron_vl.py b/vllm/model_executor/models/nemotron_vl.py index a80a61455f45..a57668b21fb8 100644 --- a/vllm/model_executor/models/nemotron_vl.py +++ b/vllm/model_executor/models/nemotron_vl.py @@ -34,8 +34,8 @@ from vllm.multimodal.image import convert_image_mode from vllm.multimodal.processing import PromptUpdateDetails from vllm.sequence import IntermediateTensors +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.processor import cached_image_processor_from_config -from vllm.transformers_utils.tokenizers import TokenizerLike from .interfaces import ( MultiModalEmbeddings, diff --git a/vllm/model_executor/models/opencua.py b/vllm/model_executor/models/opencua.py index 443b51e10b38..433891866337 100644 --- a/vllm/model_executor/models/opencua.py +++ b/vllm/model_executor/models/opencua.py @@ -31,7 +31,7 @@ PromptReplacement, PromptUpdate, ) -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from .qwen2_5_vl import ( Qwen2_5_VisionTransformer as OpenCUAVisionTransformer, diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index 069a6d2ab6d9..54bde75cc013 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -59,8 +59,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors +from vllm.tokenizers import MistralTokenizer from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config -from vllm.transformers_utils.tokenizers import MistralTokenizer from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 1dc0af7a483c..8fbd89622394 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -91,7 +91,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import ( diff --git a/vllm/model_executor/models/skyworkr1v.py b/vllm/model_executor/models/skyworkr1v.py index 827413cafe3d..55c25ce6190f 100644 --- a/vllm/model_executor/models/skyworkr1v.py +++ b/vllm/model_executor/models/skyworkr1v.py @@ -47,7 +47,7 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP diff --git a/vllm/model_executor/models/step3_vl.py b/vllm/model_executor/models/step3_vl.py index 70a82cce4b82..3e55ada0ed2e 100644 --- a/vllm/model_executor/models/step3_vl.py +++ b/vllm/model_executor/models/step3_vl.py @@ -43,8 +43,8 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder from vllm.sequence import IntermediateTensors +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.configs import Step3VisionEncoderConfig -from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 427f3be5a263..0a39ea7ef5bf 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -51,8 +51,8 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors +from vllm.tokenizers import MistralTokenizer from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config -from vllm.transformers_utils.tokenizers import MistralTokenizer from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription from .utils import init_vllm_registered_model, maybe_prefix diff --git a/vllm/multimodal/processing.py b/vllm/multimodal/processing.py index ea86f5f095d8..aab657b24ba2 100644 --- a/vllm/multimodal/processing.py +++ b/vllm/multimodal/processing.py @@ -23,9 +23,9 @@ from typing_extensions import TypeVar, assert_never from vllm.logger import init_logger +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.processor import cached_processor_from_config from vllm.transformers_utils.tokenizer import decode_tokens, encode_tokens -from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils.collection_utils import flatten_2d_lists, full_groupby from vllm.utils.func_utils import get_allowed_kwarg_only_overrides from vllm.utils.jsontree import JSONTree, json_map_leaves diff --git a/vllm/multimodal/registry.py b/vllm/multimodal/registry.py index ed1a8bce5f0e..ee90570b24aa 100644 --- a/vllm/multimodal/registry.py +++ b/vllm/multimodal/registry.py @@ -6,8 +6,8 @@ from vllm.config.multimodal import BaseDummyOptions from vllm.logger import init_logger +from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.tokenizer import cached_tokenizer_from_config -from vllm.transformers_utils.tokenizers import TokenizerLike from .cache import BaseMultiModalProcessorCache from .processing import ( diff --git a/vllm/reasoning/abs_reasoning_parsers.py b/vllm/reasoning/abs_reasoning_parsers.py index fb91885f6d5e..4a04292be009 100644 --- a/vllm/reasoning/abs_reasoning_parsers.py +++ b/vllm/reasoning/abs_reasoning_parsers.py @@ -19,7 +19,7 @@ DeltaMessage, ResponsesRequest, ) - from vllm.transformers_utils.tokenizers import TokenizerLike + from vllm.tokenizers import TokenizerLike else: ChatCompletionRequest = Any DeltaMessage = Any diff --git a/vllm/reasoning/basic_parsers.py b/vllm/reasoning/basic_parsers.py index 803696eea6b9..35084c0e7cc8 100644 --- a/vllm/reasoning/basic_parsers.py +++ b/vllm/reasoning/basic_parsers.py @@ -7,7 +7,7 @@ from vllm.entrypoints.openai.protocol import DeltaMessage from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike if TYPE_CHECKING: from vllm.entrypoints.openai.protocol import ( diff --git a/vllm/reasoning/minimax_m2_reasoning_parser.py b/vllm/reasoning/minimax_m2_reasoning_parser.py index 4bd932074361..138d1b4e6dac 100644 --- a/vllm/reasoning/minimax_m2_reasoning_parser.py +++ b/vllm/reasoning/minimax_m2_reasoning_parser.py @@ -11,7 +11,7 @@ from vllm.logger import init_logger from vllm.reasoning.abs_reasoning_parsers import ReasoningParser from vllm.reasoning.basic_parsers import BaseThinkingReasoningParser -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike logger = init_logger(__name__) diff --git a/vllm/reasoning/mistral_reasoning_parser.py b/vllm/reasoning/mistral_reasoning_parser.py index af6d179bf6d0..b61e50c188f8 100644 --- a/vllm/reasoning/mistral_reasoning_parser.py +++ b/vllm/reasoning/mistral_reasoning_parser.py @@ -6,7 +6,7 @@ from vllm.logger import init_logger from vllm.reasoning import ReasoningParser from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer +from vllm.tokenizers import MistralTokenizer logger = init_logger(__name__) diff --git a/vllm/reasoning/olmo3_reasoning_parser.py b/vllm/reasoning/olmo3_reasoning_parser.py index baa1d8355637..2742a24a2c3e 100644 --- a/vllm/reasoning/olmo3_reasoning_parser.py +++ b/vllm/reasoning/olmo3_reasoning_parser.py @@ -9,7 +9,7 @@ import regex as re if TYPE_CHECKING: - from vllm.transformers_utils.tokenizers import TokenizerLike + from vllm.tokenizers import TokenizerLike from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, diff --git a/vllm/sampling_params.py b/vllm/sampling_params.py index 5210653bc40f..453100f2e513 100644 --- a/vllm/sampling_params.py +++ b/vllm/sampling_params.py @@ -13,7 +13,7 @@ from vllm.logger import init_logger from vllm.logits_process import LogitsProcessor -from vllm.transformers_utils.tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.v1.serial_utils import PydanticMsgspecMixin logger = init_logger(__name__) diff --git a/vllm/transformers_utils/tokenizers/__init__.py b/vllm/tokenizers/__init__.py similarity index 100% rename from vllm/transformers_utils/tokenizers/__init__.py rename to vllm/tokenizers/__init__.py diff --git a/vllm/transformers_utils/tokenizers/mistral.py b/vllm/tokenizers/mistral.py similarity index 100% rename from vllm/transformers_utils/tokenizers/mistral.py rename to vllm/tokenizers/mistral.py diff --git a/vllm/transformers_utils/tokenizers/protocol.py b/vllm/tokenizers/protocol.py similarity index 97% rename from vllm/transformers_utils/tokenizers/protocol.py rename to vllm/tokenizers/protocol.py index 4b0234614018..63bb8cc1e988 100644 --- a/vllm/transformers_utils/tokenizers/protocol.py +++ b/vllm/tokenizers/protocol.py @@ -5,15 +5,10 @@ from typing_extensions import Self, runtime_checkable -from vllm.logger import init_logger - if TYPE_CHECKING: from vllm.entrypoints.chat_utils import ChatCompletionMessageParam -logger = init_logger(__name__) - - @runtime_checkable class TokenizerLike(Protocol): @classmethod diff --git a/vllm/transformers_utils/tokenizers/registry.py b/vllm/tokenizers/registry.py similarity index 100% rename from vllm/transformers_utils/tokenizers/registry.py rename to vllm/tokenizers/registry.py diff --git a/vllm/transformers_utils/detokenizer_utils.py b/vllm/transformers_utils/detokenizer_utils.py index 57947c31d97f..e586a5d46cb8 100644 --- a/vllm/transformers_utils/detokenizer_utils.py +++ b/vllm/transformers_utils/detokenizer_utils.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from .tokenizers import TokenizerLike +from vllm.tokenizers import TokenizerLike def _replace_none_with_empty(tokens: list[str | None]): diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index bb5c25d1e3b7..53be93c73de1 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -16,11 +16,11 @@ from vllm import envs from vllm.logger import init_logger +from vllm.tokenizers import TokenizerLike, TokenizerRegistry from .config import get_sentence_transformer_tokenizer_config from .gguf_utils import get_gguf_file_path_from_hf from .repo_utils import list_filtered_repo_files -from .tokenizers import TokenizerLike, TokenizerRegistry from .utils import check_gguf_file, is_gguf, is_remote_gguf, split_remote_gguf if TYPE_CHECKING: @@ -32,11 +32,9 @@ def __getattr__(name: str): if name == "AnyTokenizer": - from .tokenizers import TokenizerLike - warnings.warn( "`vllm.transformers_utils.tokenizer.AnyTokenizer` has been moved to " - "`vllm.transformers_utils.tokenizers.TokenizerLike`. " + "`vllm.tokenizers.TokenizerLike`. " "The old name will be removed in v0.13.", DeprecationWarning, stacklevel=2, @@ -217,7 +215,7 @@ def get_tokenizer( tokenizer: TokenizerLike if tokenizer_mode == "mistral": - from .tokenizers import MistralTokenizer + from vllm.tokenizers import MistralTokenizer logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}") tokenizer = MistralTokenizer.from_pretrained( diff --git a/vllm/transformers_utils/tokenizer_base.py b/vllm/transformers_utils/tokenizer_base.py index b658320d9d67..78fb6edc8b9e 100644 --- a/vllm/transformers_utils/tokenizer_base.py +++ b/vllm/transformers_utils/tokenizer_base.py @@ -6,11 +6,11 @@ def __getattr__(name: str): if name == "TokenizerBase": - from .tokenizers import TokenizerLike + from vllm.tokenizers import TokenizerLike warnings.warn( "`vllm.transformers_utils.tokenizer_base.TokenizerBase` has been " - "moved to `vllm.transformers_utils.tokenizers.TokenizerLike`. " + "moved to `vllm.tokenizers.TokenizerLike`. " "The old name will be removed in v0.13.", DeprecationWarning, stacklevel=2, @@ -18,11 +18,11 @@ def __getattr__(name: str): return TokenizerLike if name == "TokenizerRegistry": - from .tokenizers import TokenizerRegistry + from vllm.tokenizers import TokenizerRegistry warnings.warn( "`vllm.transformers_utils.tokenizer_base.TokenizerRegistry` has been " - "moved to `vllm.transformers_utils.tokenizers.TokenizerRegistry`. " + "moved to `vllm.tokenizers.TokenizerRegistry`. " "The old name will be removed in v0.13.", DeprecationWarning, stacklevel=2, diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 487a811f9af1..336d3e9fa1d2 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -26,10 +26,10 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask +from vllm.tokenizers import TokenizerLike from vllm.tracing import init_tracer from vllm.transformers_utils.config import maybe_register_config_serialize_by_value from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs -from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.usage.usage_lib import UsageContext from vllm.utils.async_utils import cancel_task_threadsafe from vllm.utils.collection_utils import as_list diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 7c67d2421b04..e6a94f4e3de5 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -19,8 +19,7 @@ from vllm.multimodal.utils import argsort_mm_positions from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizers import TokenizerLike -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer +from vllm.tokenizers import MistralTokenizer, TokenizerLike from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.v1.engine import EngineCoreRequest from vllm.v1.metrics.stats import MultiModalCacheStats diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 9812197cec2c..a3bde7ba8d64 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -23,9 +23,9 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask +from vllm.tokenizers import TokenizerLike from vllm.tracing import init_tracer from vllm.transformers_utils.tokenizer import init_tokenizer_from_configs -from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.usage.usage_lib import UsageContext from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.core_client import EngineCoreClient diff --git a/vllm/v1/engine/output_processor.py b/vllm/v1/engine/output_processor.py index f6308d63e82c..e85fbb4ee0fb 100644 --- a/vllm/v1/engine/output_processor.py +++ b/vllm/v1/engine/output_processor.py @@ -15,8 +15,8 @@ RequestOutput, ) from vllm.sampling_params import RequestOutputKind +from vllm.tokenizers import TokenizerLike from vllm.tracing import SpanAttributes, SpanKind, Tracer, extract_trace_context -from vllm.transformers_utils.tokenizers import TokenizerLike from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason from vllm.v1.engine.detokenizer import IncrementalDetokenizer diff --git a/vllm/v1/structured_output/backend_types.py b/vllm/v1/structured_output/backend_types.py index 230b47470d6b..5c09b7b0634f 100644 --- a/vllm/v1/structured_output/backend_types.py +++ b/vllm/v1/structured_output/backend_types.py @@ -10,7 +10,7 @@ import torch from vllm.config import VllmConfig - from vllm.transformers_utils.tokenizers import TokenizerLike + from vllm.tokenizers import TokenizerLike else: VllmConfig = object TokenizerLike = object diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index c9f2dc07da78..f8a2df43dd90 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -10,7 +10,7 @@ import vllm.envs from vllm.logger import init_logger from vllm.sampling_params import SamplingParams -from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer +from vllm.tokenizers import MistralTokenizer from vllm.utils.import_utils import LazyLoader from vllm.v1.structured_output.backend_types import ( StructuredOutputBackend, diff --git a/vllm/v1/structured_output/utils.py b/vllm/v1/structured_output/utils.py index 4602205e5aa6..ae42b33f80f8 100644 --- a/vllm/v1/structured_output/utils.py +++ b/vllm/v1/structured_output/utils.py @@ -24,7 +24,7 @@ import transformers.models.gpt2.tokenization_gpt2 as tokenization_gpt2 import xgrammar as xgr - from vllm.transformers_utils.tokenizers import TokenizerLike + from vllm.tokenizers import TokenizerLike from vllm.v1.worker.gpu_input_batch import InputBatch else: xgr = LazyLoader("xgr", globals(), "xgrammar") From c59476ab457e5298c64458c8eff7c0200c170b40 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 05:04:43 +0000 Subject: [PATCH 25/31] Unnecessary `runtime_checkable` Signed-off-by: DarkLight1337 --- vllm/tokenizers/protocol.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vllm/tokenizers/protocol.py b/vllm/tokenizers/protocol.py index 63bb8cc1e988..58a1a7c23f21 100644 --- a/vllm/tokenizers/protocol.py +++ b/vllm/tokenizers/protocol.py @@ -3,13 +3,12 @@ from typing import TYPE_CHECKING, Any, Protocol -from typing_extensions import Self, runtime_checkable +from typing_extensions import Self if TYPE_CHECKING: from vllm.entrypoints.chat_utils import ChatCompletionMessageParam -@runtime_checkable class TokenizerLike(Protocol): @classmethod def from_pretrained( From 9d974a3c796ad73a96264dd53f15c7e8e9ead7d5 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 05:20:10 +0000 Subject: [PATCH 26/31] Update import Signed-off-by: DarkLight1337 --- vllm/transformers_utils/tokenizer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 53be93c73de1..87d5cc2b483f 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -16,7 +16,7 @@ from vllm import envs from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike, TokenizerRegistry +from vllm.tokenizers import MistralTokenizer, TokenizerLike, TokenizerRegistry from .config import get_sentence_transformer_tokenizer_config from .gguf_utils import get_gguf_file_path_from_hf @@ -215,8 +215,6 @@ def get_tokenizer( tokenizer: TokenizerLike if tokenizer_mode == "mistral": - from vllm.tokenizers import MistralTokenizer - logger.debug_once(f"Loading MistralTokenizer from {tokenizer_name}") tokenizer = MistralTokenizer.from_pretrained( str(tokenizer_name), revision=revision From 15bf2a03cc8c53bfd78a49e6a60b5f3308262b32 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 06:08:50 +0000 Subject: [PATCH 27/31] Avoid conflict with `tokenizers` package Signed-off-by: DarkLight1337 --- .buildkite/test-amd.yaml | 4 ++-- .buildkite/test-pipeline.yaml | 4 ++-- tests/tokenizers/__init__.py | 0 tests/tokenizers_/__init__.py | 4 ++++ tests/{tokenizers => tokenizers_}/test_basic.py | 0 tests/{tokenizers => tokenizers_}/test_cached_tokenizer.py | 0 tests/{tokenizers => tokenizers_}/test_detokenize.py | 0 tests/{tokenizers => tokenizers_}/test_mistral.py | 0 tests/{tokenizers => tokenizers_}/test_registry.py | 0 tools/pre_commit/check_pickle_imports.py | 2 +- 10 files changed, 9 insertions(+), 5 deletions(-) delete mode 100644 tests/tokenizers/__init__.py create mode 100644 tests/tokenizers_/__init__.py rename tests/{tokenizers => tokenizers_}/test_basic.py (100%) rename tests/{tokenizers => tokenizers_}/test_cached_tokenizer.py (100%) rename tests/{tokenizers => tokenizers_}/test_detokenize.py (100%) rename tests/{tokenizers => tokenizers_}/test_mistral.py (100%) rename tests/{tokenizers => tokenizers_}/test_registry.py (100%) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 30e5dd9a3453..4d98ee40a4bb 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -316,7 +316,7 @@ steps: source_file_dependencies: - vllm/ - tests/engine - - tests/tokenizers + - tests/tokenizers_ - tests/test_sequence - tests/test_config - tests/test_logger @@ -324,7 +324,7 @@ steps: commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py # OOM in the CI unless we run this separately - - pytest -v -s tokenizers + - pytest -v -s tokenizers_ - label: V1 Test e2e + engine # 30min timeout_in_minutes: 45 diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index ad6e165dca81..16d490754958 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -282,7 +282,7 @@ steps: source_file_dependencies: - vllm/ - tests/engine - - tests/tokenizers + - tests/tokenizers_ - tests/test_sequence - tests/test_config - tests/test_logger @@ -290,7 +290,7 @@ steps: commands: - pytest -v -s engine test_sequence.py test_config.py test_logger.py test_vllm_port.py # OOM in the CI unless we run this separately - - pytest -v -s tokenizers + - pytest -v -s tokenizers_ - label: V1 Test e2e + engine # 30min timeout_in_minutes: 45 diff --git a/tests/tokenizers/__init__.py b/tests/tokenizers/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/tokenizers_/__init__.py b/tests/tokenizers_/__init__.py new file mode 100644 index 000000000000..a5d7f4b03103 --- /dev/null +++ b/tests/tokenizers_/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# NOTE: Since CI runs the tests from the `tests` directory, it is necessary to rename +# this module to avoid conflicting with HF's `tokenizers` package diff --git a/tests/tokenizers/test_basic.py b/tests/tokenizers_/test_basic.py similarity index 100% rename from tests/tokenizers/test_basic.py rename to tests/tokenizers_/test_basic.py diff --git a/tests/tokenizers/test_cached_tokenizer.py b/tests/tokenizers_/test_cached_tokenizer.py similarity index 100% rename from tests/tokenizers/test_cached_tokenizer.py rename to tests/tokenizers_/test_cached_tokenizer.py diff --git a/tests/tokenizers/test_detokenize.py b/tests/tokenizers_/test_detokenize.py similarity index 100% rename from tests/tokenizers/test_detokenize.py rename to tests/tokenizers_/test_detokenize.py diff --git a/tests/tokenizers/test_mistral.py b/tests/tokenizers_/test_mistral.py similarity index 100% rename from tests/tokenizers/test_mistral.py rename to tests/tokenizers_/test_mistral.py diff --git a/tests/tokenizers/test_registry.py b/tests/tokenizers_/test_registry.py similarity index 100% rename from tests/tokenizers/test_registry.py rename to tests/tokenizers_/test_registry.py diff --git a/tools/pre_commit/check_pickle_imports.py b/tools/pre_commit/check_pickle_imports.py index b738d8437cc7..2bb468da68c2 100644 --- a/tools/pre_commit/check_pickle_imports.py +++ b/tools/pre_commit/check_pickle_imports.py @@ -27,7 +27,7 @@ "vllm/distributed/device_communicators/shm_broadcast.py", "vllm/distributed/device_communicators/shm_object_storage.py", "vllm/utils/hashing.py", - "tests/tokenizers/test_cached_tokenizer.py", + "tests/tokenizers_/test_cached_tokenizer.py", "tests/utils_/test_hashing.py", "benchmarks/kernels/graph_machete_bench.py", "benchmarks/kernels/benchmark_lora.py", From 6b558be3e825074985b993b8486ba0f0f952ced2 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 06:38:09 +0000 Subject: [PATCH 28/31] Don't run type validation on internal structures Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 41 ++++++----------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 48b2248b4e9d..0cb807bad771 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -7,13 +7,14 @@ import traceback from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field from http import HTTPStatus from typing import Any, ClassVar, Generic, TypeAlias, TypeVar import numpy as np import torch from fastapi import Request -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter +from pydantic import ConfigDict, TypeAdapter from starlette.datastructures import Headers from typing_extensions import TypeIs @@ -184,19 +185,19 @@ def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]: RequestT = TypeVar("RequestT", bound=AnyRequest) -class RequestProcessingMixin(BaseModel): +@dataclass +class RequestProcessingMixin: """ Mixin for request processing, handling prompt preparation and engine input. """ - request_prompts: Sequence[RequestPrompt] | None = [] - engine_prompts: list[EngineTokensPrompt] | None = [] - - model_config = ConfigDict(arbitrary_types_allowed=True) + request_prompts: Sequence[RequestPrompt] | None = field(default_factory=list) + engine_prompts: list[EngineTokensPrompt] | None = field(default_factory=list) -class ResponseGenerationMixin(BaseModel): +@dataclass +class ResponseGenerationMixin: """ Mixin for response generation, managing result generators and final batch results. @@ -205,37 +206,25 @@ class ResponseGenerationMixin(BaseModel): result_generator: ( AsyncGenerator[tuple[int, RequestOutput | PoolingRequestOutput], None] | None ) = None - final_res_batch: list[RequestOutput | PoolingRequestOutput] = Field( + final_res_batch: list[RequestOutput | PoolingRequestOutput] = field( default_factory=list ) model_config = ConfigDict(arbitrary_types_allowed=True) -class ServeContext( - RequestProcessingMixin, - ResponseGenerationMixin, - BaseModel, - Generic[RequestT], -): +class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[RequestT]): # Shared across all requests request: RequestT raw_request: Request | None = None model_name: str request_id: str - created_time: int = Field(default_factory=lambda: int(time.time())) + created_time: int = field(default_factory=lambda: int(time.time())) lora_request: LoRARequest | None = None # Shared across most requests tokenizer: TokenizerLike | None = None - # `protected_namespaces` resolves Pydantic v2's warning - # on conflict with protected namespace "model_" - model_config = ConfigDict( - protected_namespaces=(), - arbitrary_types_allowed=True, - ) - ClassificationServeContext = ServeContext[ClassificationRequest] @@ -245,14 +234,6 @@ class EmbeddingServeContext(ServeContext[EmbeddingRequest]): chat_template_content_format: ChatTemplateContentFormatOption -# Used to resolve the Pydantic error related to -# forward reference of MultiModalDataDict in TokensPrompt -RequestProcessingMixin.model_rebuild() -ServeContext.model_rebuild() -ClassificationServeContext.model_rebuild() -EmbeddingServeContext.model_rebuild() - - class OpenAIServing: request_id_prefix: ClassVar[str] = """ A short string prepended to every request’s ID (e.g. "embd", "classify") From 767f2c85227a1cd67ae6f2896aee0bd8f3115d2a Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 06:39:36 +0000 Subject: [PATCH 29/31] kw only Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 0cb807bad771..e7a632e02510 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -185,7 +185,7 @@ def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]: RequestT = TypeVar("RequestT", bound=AnyRequest) -@dataclass +@dataclass(kw_only=True) class RequestProcessingMixin: """ Mixin for request processing, @@ -196,7 +196,7 @@ class RequestProcessingMixin: engine_prompts: list[EngineTokensPrompt] | None = field(default_factory=list) -@dataclass +@dataclass(kw_only=True) class ResponseGenerationMixin: """ Mixin for response generation, @@ -213,6 +213,7 @@ class ResponseGenerationMixin: model_config = ConfigDict(arbitrary_types_allowed=True) +@dataclass(kw_only=True) class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[RequestT]): # Shared across all requests request: RequestT @@ -226,9 +227,12 @@ class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[Requ tokenizer: TokenizerLike | None = None -ClassificationServeContext = ServeContext[ClassificationRequest] +@dataclass(kw_only=True) +class ClassificationServeContext(ServeContext[ClassificationRequest]): + pass +@dataclass(kw_only=True) class EmbeddingServeContext(ServeContext[EmbeddingRequest]): chat_template: str | None = None chat_template_content_format: ChatTemplateContentFormatOption From 02c18573012c9a531e6712f714a908a4c10c30df Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 06:47:07 +0000 Subject: [PATCH 30/31] Fix mypy Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_classification.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/serving_classification.py b/vllm/entrypoints/openai/serving_classification.py index 167ee152fece..3b973eb125a8 100644 --- a/vllm/entrypoints/openai/serving_classification.py +++ b/vllm/entrypoints/openai/serving_classification.py @@ -221,7 +221,7 @@ async def create_classify( def _create_pooling_params( self, - ctx: ClassificationServeContext, + ctx: ServeContext[ClassificationRequest], ) -> PoolingParams | ErrorResponse: pooling_params = super()._create_pooling_params(ctx) if isinstance(pooling_params, ErrorResponse): From a5fbb67f6f69a00d272f5b9350ff59e62121ab91 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 29 Nov 2025 09:13:16 +0000 Subject: [PATCH 31/31] Fix test Signed-off-by: DarkLight1337 --- tests/tokenizers_/test_registry.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index ad085eedefdb..1eb19a0996dd 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -21,8 +21,8 @@ def eos_token_id(self) -> int: def test_customized_tokenizer(): TokenizerRegistry.register( "test_tokenizer", - "tests.tokenizers.test_registry", - "TestTokenizer", + __name__, + TestTokenizer.__name__, ) tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer")