[Frontend] Introduce Renderer for processing chat messages (using ModelConfig)

benchmarks/backend_request_func.py

@@ -620,7 +620,7 @@ def get_tokenizer(
         kwargs["use_fast"] = False
     if tokenizer_mode == "mistral":
         try:
-            from vllm.tokenizers import MistralTokenizer
+            from vllm.tokenizers.mistral import MistralTokenizer
         except ImportError as e:
             raise ImportError(
                 "MistralTokenizer requires vllm package.\n"

tests/entrypoints/openai/test_chat_template.py

@@ -4,9 +4,11 @@
 import pytest
 
 from vllm.config import ModelConfig
-from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
+from vllm.entrypoints.chat_utils import load_chat_template
 from vllm.entrypoints.openai.protocol import ChatCompletionRequest
+from vllm.renderers.hf import safe_apply_chat_template
 from vllm.tokenizers import get_tokenizer
+from vllm.tokenizers.hf import CachedHfTokenizer
 
 from ...models.registry import HF_EXAMPLE_MODELS
 from ...utils import VLLM_PATH
@@ -123,7 +125,8 @@ def test_get_gen_prompt(
 
     # Initialize the tokenizer
     tokenizer = get_tokenizer(
-        tokenizer_name=model_config.tokenizer,
+        model_config.tokenizer,
+        tokenizer_cls=CachedHfTokenizer,
         trust_remote_code=model_config.trust_remote_code,
     )
     template_content = load_chat_template(chat_template=template)
@@ -139,14 +142,15 @@ def test_get_gen_prompt(
     )
 
     # Call the function and get the result
-    result = apply_hf_chat_template(
-        tokenizer=tokenizer,
-        conversation=mock_request.messages,
-        chat_template=mock_request.chat_template or template_content,
-        model_config=model_config,
+    result = safe_apply_chat_template(
+        model_config,
+        tokenizer,
+        mock_request.messages,
         tools=None,
+        chat_template=mock_request.chat_template or template_content,
         add_generation_prompt=mock_request.add_generation_prompt,
         continue_final_message=mock_request.continue_final_message,
+        tokenize=False,
     )
 
     # Test assertion

tests/entrypoints/openai/test_serving_engine.py

@@ -10,7 +10,7 @@
 from vllm.config import ModelConfig
 from vllm.entrypoints.openai.serving_engine import OpenAIServing
 from vllm.entrypoints.openai.serving_models import OpenAIServingModels
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 
 
 @pytest.fixture()

tests/entrypoints/test_chat_utils.py

@@ -29,7 +29,8 @@
     encode_image_base64,
     encode_video_base64,
 )
-from vllm.tokenizers import MistralTokenizer, get_tokenizer
+from vllm.tokenizers import get_tokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 from vllm.utils.serial_utils import tensor2base64
 
 from ..models.registry import HF_EXAMPLE_MODELS

tests/models/language/generation/test_mistral.py

@@ -10,7 +10,7 @@
     MistralToolParser,
 )
 from vllm.sampling_params import SamplingParams
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 
 from ...utils import check_logprobs_close
 

tests/models/multimodal/generation/test_voxtral.py

@@ -9,7 +9,7 @@
 from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk
 from mistral_common.protocol.instruct.messages import UserMessage
 
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 
 from ....conftest import AudioTestAssets
 from ....utils import RemoteOpenAIServer

tests/models/multimodal/processing/test_common.py

@@ -22,11 +22,8 @@
 from vllm.multimodal.cache import MultiModalProcessorOnlyCache
 from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal
 from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext
-from vllm.tokenizers import (
-    MistralTokenizer,
-    TokenizerLike,
-    cached_tokenizer_from_config,
-)
+from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config
+from vllm.tokenizers.mistral import MistralTokenizer
 
 from ....multimodal.utils import random_audio, random_image, random_video
 from ...registry import (

tests/reasoning/test_mistral_reasoning_parser.py

@@ -5,7 +5,7 @@
 
 from tests.reasoning.utils import run_reasoning_extraction_mistral
 from vllm.reasoning import ReasoningParser, ReasoningParserManager
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 
 parser_name = "mistral"
 

tests/reasoning/utils.py

@@ -4,7 +4,7 @@
 
 from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage
 from vllm.reasoning import ReasoningParser
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 
 
 class StreamingReasoningReconstructor:

tests/test_inputs.py

@@ -7,7 +7,6 @@
 from vllm.inputs import zip_enc_dec_prompts
 from vllm.inputs.parse import parse_raw_prompts
 from vllm.inputs.preprocess import InputPreprocessor
-from vllm.tokenizers import init_tokenizer_from_config
 
 pytestmark = pytest.mark.cpu_test
 
@@ -108,10 +107,10 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs):
 )
 def test_preprocessor_always_mm_code_path(model_id, prompt):
     model_config = ModelConfig(model=model_id)
-    tokenizer = init_tokenizer_from_config(model_config)
-    input_preprocessor = InputPreprocessor(model_config, tokenizer)
+    input_preprocessor = InputPreprocessor(model_config)
 
     # HF processor adds sep token
+    tokenizer = input_preprocessor.get_tokenizer()
     sep_token_id = tokenizer.vocab[tokenizer.sep_token]
 
     processed_inputs = input_preprocessor.preprocess(prompt)

tests/tokenizers_/test_detokenize.py

@@ -8,7 +8,7 @@
 from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast
 
 from vllm.sampling_params import SamplingParams
-from vllm.tokenizers import MistralTokenizer
+from vllm.tokenizers.mistral import MistralTokenizer
 from vllm.v1.engine import EngineCoreRequest
 from vllm.v1.engine.detokenizer import (
     FastIncrementalDetokenizer,

tests/tokenizers_/test_registry.py

@@ -2,7 +2,14 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 from pathlib import Path
 
-from vllm.tokenizers import TokenizerLike, TokenizerRegistry, get_tokenizer
+import pytest
+
+from vllm.tokenizers import TokenizerLike
+from vllm.tokenizers.registry import (
+    TokenizerRegistry,
+    get_tokenizer,
+    resolve_tokenizer_args,
+)
 
 
 class TestTokenizer(TokenizerLike):
@@ -40,10 +47,33 @@ def is_fast(self) -> bool:
         return True
 
 
+@pytest.mark.parametrize("runner_type", ["generate", "pooling"])
+def test_resolve_tokenizer_args_idempotent(runner_type):
+    tokenizer_name = "facebook/opt-125m"
+
+    tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs = (
+        resolve_tokenizer_args(
+            tokenizer_name,
+            runner_type=runner_type,
+        )
+    )
+
+    assert (
+        tokenizer_mode,
+        tokenizer_name,
+        tokenizer_args,
+        tokenizer_kwargs,
+    ) == resolve_tokenizer_args(
+        tokenizer_name,
+        *tokenizer_args,
+        **tokenizer_kwargs,
+    )
+
+
 def test_customized_tokenizer():
     TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__)
 
-    tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer", "abc")
+    tokenizer = TokenizerRegistry.load_tokenizer("test_tokenizer", "abc")
     assert isinstance(tokenizer, TestTokenizer)
     assert tokenizer.path_or_repo_id == "abc"
     assert tokenizer.bos_token_id == 0

tests/tool_use/test_mistral_tool_parser.py

@@ -13,12 +13,9 @@
 
 from vllm.entrypoints.openai.protocol import DeltaMessage, DeltaToolCall
 from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolParser
-from vllm.tokenizers import (
-    MistralTokenizer,
-    TokenizerLike,
-    get_tokenizer,
-)
+from vllm.tokenizers import TokenizerLike, get_tokenizer
 from vllm.tokenizers.detokenizer_utils import detokenize_incrementally
+from vllm.tokenizers.mistral import MistralTokenizer
 
 
 @pytest.fixture(scope="module")

tests/v1/engine/test_process_multi_modal_uuids.py

@@ -5,7 +5,13 @@
 
 from vllm.assets.image import ImageAsset
 from vllm.assets.video import VideoAsset
-from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig
+from vllm.config import (
+    CacheConfig,
+    DeviceConfig,
+    ModelConfig,
+    MultiModalConfig,
+    VllmConfig,
+)
 from vllm.sampling_params import SamplingParams
 from vllm.v1.engine import input_processor as input_processor_mod
 from vllm.v1.engine.input_processor import InputProcessor
@@ -44,27 +50,22 @@ def _mock_input_processor(
     monkeypatch.setattr(VllmConfig, "__post_init__", lambda self: None, raising=True)
 
     model_config = ModelConfig(
+        tokenizer="dummy",
         skip_tokenizer_init=True,
         max_model_len=128,
         mm_processor_cache_gb=mm_cache_gb,
         generation_config="vllm",
-        tokenizer="dummy",
     )
+    model_config.runner_type = "generate"
+    model_config.multimodal_config = MultiModalConfig(mm_processor_cache_gb=mm_cache_gb)
 
-    # Minimal multimodal_config to satisfy references in
-    # Processor.process_inputs.
-    class _MockMMConfig:
-        def __init__(self, gb: float):
-            self.mm_processor_cache_gb = gb
-
-    model_config.multimodal_config = _MockMMConfig(mm_cache_gb)  # type: ignore[attr-defined]
     vllm_config = VllmConfig(
         model_config=model_config,
         cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching),
         device_config=DeviceConfig(device="cpu"),
     )
 
-    return InputProcessor(vllm_config, tokenizer=None)
+    return InputProcessor(vllm_config)
 
 
 def test_multi_modal_uuids_length_mismatch_raises(monkeypatch):

tools/pre_commit/mypy.py

@@ -35,6 +35,7 @@
     "vllm/multimodal",
     "vllm/platforms",
     "vllm/plugins",
+    "vllm/renderers",
     "vllm/tokenizers",
     "vllm/transformers_utils",
     "vllm/triton_utils",

vllm/engine/protocol.py

@@ -11,9 +11,9 @@
 from vllm.outputs import PoolingRequestOutput, RequestOutput
 from vllm.plugins.io_processors import IOProcessor
 from vllm.pooling_params import PoolingParams
+from vllm.renderers import RendererLike
 from vllm.sampling_params import SamplingParams
 from vllm.tasks import SupportedTask
-from vllm.tokenizers import TokenizerLike
 from vllm.v1.engine import EngineCoreRequest
 from vllm.v1.engine.input_processor import InputProcessor
 
@@ -26,6 +26,10 @@ class EngineClient(ABC):
     input_processor: InputProcessor
     io_processor: IOProcessor | None
 
+    @property
+    @abstractmethod
+    def renderer(self) -> RendererLike: ...
+
     @property
     @abstractmethod
     def is_running(self) -> bool: ...
@@ -84,11 +88,6 @@ async def abort(self, request_id: str | Iterable[str]) -> None:
         """
         ...
 
-    @abstractmethod
-    async def get_tokenizer(self) -> TokenizerLike:
-        """Get the tokenizer"""
-        ...
-
     @abstractmethod
     async def is_tracing_enabled(self) -> bool: ...