From f73bafb974560d0e62d97166587709e782d31558 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 08:15:13 +0000 Subject: [PATCH 01/65] [Renderer] Introduce Renderer Signed-off-by: DarkLight1337 --- benchmarks/backend_request_func.py | 2 +- .../entrypoints/openai/test_chat_template.py | 16 +- .../entrypoints/openai/test_serving_engine.py | 2 +- tests/entrypoints/test_chat_utils.py | 3 +- .../language/generation/test_mistral.py | 2 +- .../multimodal/generation/test_voxtral.py | 2 +- .../multimodal/processing/test_common.py | 7 +- tests/test_inputs.py | 5 +- tests/tokenizers_/test_registry.py | 2 +- .../engine/test_process_multi_modal_uuids.py | 21 +- vllm/engine/protocol.py | 5 + vllm/entrypoints/chat_utils.py | 530 +--------------- vllm/entrypoints/context.py | 13 +- vllm/entrypoints/llm.py | 73 +-- vllm/entrypoints/openai/serving_chat.py | 5 +- vllm/entrypoints/openai/serving_engine.py | 148 +---- vllm/entrypoints/openai/serving_models.py | 1 + vllm/entrypoints/openai/serving_responses.py | 12 +- .../openai/tool_parsers/hermes_tool_parser.py | 3 +- .../openai/tool_parsers/jamba_tool_parser.py | 3 +- .../tool_parsers/mistral_tool_parser.py | 6 +- vllm/entrypoints/pooling/classify/serving.py | 6 +- vllm/entrypoints/pooling/embed/serving.py | 7 +- vllm/entrypoints/pooling/pooling/serving.py | 13 +- vllm/entrypoints/pooling/score/serving.py | 6 +- vllm/entrypoints/serve/tokenize/serving.py | 7 +- vllm/entrypoints/utils.py | 2 +- vllm/inputs/preprocess.py | 19 +- vllm/model_executor/models/pixtral.py | 3 +- vllm/model_executor/models/voxtral.py | 3 +- vllm/reasoning/mistral_reasoning_parser.py | 2 +- vllm/renderers/__init__.py | 7 + vllm/renderers/deepseekv32.py | 119 ++++ vllm/renderers/hf.py | 596 ++++++++++++++++++ vllm/renderers/mistral.py | 147 +++++ vllm/renderers/protocol.py | 53 ++ vllm/renderers/registry.py | 110 ++++ vllm/renderers/terratorch.py | 80 +++ vllm/tokenizers/__init__.py | 6 - vllm/tokenizers/deepseekv32.py | 31 +- vllm/tokenizers/hf.py | 21 +- vllm/tokenizers/mistral.py | 7 +- vllm/tokenizers/registry.py | 120 +--- vllm/v1/engine/async_llm.py | 19 +- vllm/v1/engine/input_processor.py | 13 +- vllm/v1/engine/llm_engine.py | 19 +- vllm/v1/structured_output/backend_xgrammar.py | 2 +- 47 files changed, 1363 insertions(+), 916 deletions(-) create mode 100644 vllm/renderers/__init__.py create mode 100644 vllm/renderers/deepseekv32.py create mode 100644 vllm/renderers/hf.py create mode 100644 vllm/renderers/mistral.py create mode 100644 vllm/renderers/protocol.py create mode 100644 vllm/renderers/registry.py create mode 100644 vllm/renderers/terratorch.py diff --git a/benchmarks/backend_request_func.py b/benchmarks/backend_request_func.py index d69d74ca61f5..831b76b66e09 100644 --- a/benchmarks/backend_request_func.py +++ b/benchmarks/backend_request_func.py @@ -620,7 +620,7 @@ def get_tokenizer( kwargs["use_fast"] = False if tokenizer_mode == "mistral": try: - from vllm.tokenizers import MistralTokenizer + from vllm.tokenizers.mistral import MistralTokenizer except ImportError as e: raise ImportError( "MistralTokenizer requires vllm package.\n" diff --git a/tests/entrypoints/openai/test_chat_template.py b/tests/entrypoints/openai/test_chat_template.py index 77087ac21ea8..4db7c9576b1d 100644 --- a/tests/entrypoints/openai/test_chat_template.py +++ b/tests/entrypoints/openai/test_chat_template.py @@ -4,9 +4,11 @@ import pytest from vllm.config import ModelConfig -from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template +from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.openai.protocol import ChatCompletionRequest +from vllm.renderers.hf import safe_apply_chat_template from vllm.tokenizers import get_tokenizer +from vllm.tokenizers.hf import HfTokenizer from ...models.registry import HF_EXAMPLE_MODELS from ...utils import VLLM_PATH @@ -123,6 +125,7 @@ def test_get_gen_prompt( # Initialize the tokenizer tokenizer = get_tokenizer( + HfTokenizer, tokenizer_name=model_config.tokenizer, trust_remote_code=model_config.trust_remote_code, ) @@ -139,14 +142,15 @@ def test_get_gen_prompt( ) # Call the function and get the result - result = apply_hf_chat_template( - tokenizer=tokenizer, - conversation=mock_request.messages, - chat_template=mock_request.chat_template or template_content, - model_config=model_config, + result = safe_apply_chat_template( + model_config, + tokenizer, + mock_request.messages, tools=None, + chat_template=mock_request.chat_template or template_content, add_generation_prompt=mock_request.add_generation_prompt, continue_final_message=mock_request.continue_final_message, + tokenize=False, ) # Test assertion diff --git a/tests/entrypoints/openai/test_serving_engine.py b/tests/entrypoints/openai/test_serving_engine.py index 956a06dc5487..192c7cafb749 100644 --- a/tests/entrypoints/openai/test_serving_engine.py +++ b/tests/entrypoints/openai/test_serving_engine.py @@ -10,7 +10,7 @@ from vllm.config import ModelConfig from vllm.entrypoints.openai.serving_engine import OpenAIServing from vllm.entrypoints.openai.serving_models import OpenAIServingModels -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer @pytest.fixture() diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index 527322c71ae4..99a18b20c7c6 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -29,7 +29,8 @@ encode_image_base64, encode_video_base64, ) -from vllm.tokenizers import MistralTokenizer, get_tokenizer +from vllm.tokenizers import get_tokenizer +from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.serial_utils import tensor2base64 from ..models.registry import HF_EXAMPLE_MODELS diff --git a/tests/models/language/generation/test_mistral.py b/tests/models/language/generation/test_mistral.py index e2d6271e2fae..bc8bb05c284e 100644 --- a/tests/models/language/generation/test_mistral.py +++ b/tests/models/language/generation/test_mistral.py @@ -10,7 +10,7 @@ MistralToolParser, ) from vllm.sampling_params import SamplingParams -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer from ...utils import check_logprobs_close diff --git a/tests/models/multimodal/generation/test_voxtral.py b/tests/models/multimodal/generation/test_voxtral.py index 9e9087cb0fc4..0eaef49e2395 100644 --- a/tests/models/multimodal/generation/test_voxtral.py +++ b/tests/models/multimodal/generation/test_voxtral.py @@ -9,7 +9,7 @@ from mistral_common.protocol.instruct.chunk import AudioChunk, RawAudio, TextChunk from mistral_common.protocol.instruct.messages import UserMessage -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer from ....conftest import AudioTestAssets from ....utils import RemoteOpenAIServer diff --git a/tests/models/multimodal/processing/test_common.py b/tests/models/multimodal/processing/test_common.py index 2e032ac4ca52..67861ebfc44e 100644 --- a/tests/models/multimodal/processing/test_common.py +++ b/tests/models/multimodal/processing/test_common.py @@ -22,11 +22,8 @@ from vllm.multimodal.cache import MultiModalProcessorOnlyCache from vllm.multimodal.inputs import MultiModalInputs, batched_tensors_equal from vllm.multimodal.processing import BaseMultiModalProcessor, InputProcessingContext -from vllm.tokenizers import ( - MistralTokenizer, - TokenizerLike, - cached_tokenizer_from_config, -) +from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config +from vllm.tokenizers.mistral import MistralTokenizer from ....multimodal.utils import random_audio, random_image, random_video from ...registry import ( diff --git a/tests/test_inputs.py b/tests/test_inputs.py index c4339827de8b..4cd565bf33dc 100644 --- a/tests/test_inputs.py +++ b/tests/test_inputs.py @@ -7,7 +7,6 @@ from vllm.inputs import zip_enc_dec_prompts from vllm.inputs.parse import parse_raw_prompts from vllm.inputs.preprocess import InputPreprocessor -from vllm.tokenizers import init_tokenizer_from_config pytestmark = pytest.mark.cpu_test @@ -108,10 +107,10 @@ def test_zip_enc_dec_prompts(mm_processor_kwargs, expected_mm_kwargs): ) def test_preprocessor_always_mm_code_path(model_id, prompt): model_config = ModelConfig(model=model_id) - tokenizer = init_tokenizer_from_config(model_config) - input_preprocessor = InputPreprocessor(model_config, tokenizer) + input_preprocessor = InputPreprocessor(model_config) # HF processor adds sep token + tokenizer = input_preprocessor.get_tokenizer() sep_token_id = tokenizer.vocab[tokenizer.sep_token] processed_inputs = input_preprocessor.preprocess(prompt) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index 7e795350d64c..f2001bc19a37 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -43,7 +43,7 @@ def is_fast(self) -> bool: def test_customized_tokenizer(): TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__) - tokenizer = TokenizerRegistry.get_tokenizer("test_tokenizer", "abc") + tokenizer = TokenizerRegistry.init_tokenizer("test_tokenizer", "abc") assert isinstance(tokenizer, TestTokenizer) assert tokenizer.path_or_repo_id == "abc" assert tokenizer.bos_token_id == 0 diff --git a/tests/v1/engine/test_process_multi_modal_uuids.py b/tests/v1/engine/test_process_multi_modal_uuids.py index 1b11b8af49d1..46a79790a212 100644 --- a/tests/v1/engine/test_process_multi_modal_uuids.py +++ b/tests/v1/engine/test_process_multi_modal_uuids.py @@ -5,7 +5,14 @@ from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset -from vllm.config import CacheConfig, DeviceConfig, ModelConfig, VllmConfig +from vllm.config import ( + CacheConfig, + DeviceConfig, + ModelConfig, + MultiModalConfig, + VllmConfig, +) +from vllm.renderers.terratorch import TerratorchRenderer from vllm.sampling_params import SamplingParams from vllm.v1.engine import input_processor as input_processor_mod from vllm.v1.engine.input_processor import InputProcessor @@ -48,23 +55,15 @@ def _mock_input_processor( max_model_len=128, mm_processor_cache_gb=mm_cache_gb, generation_config="vllm", - tokenizer="dummy", ) + model_config.multimodal_config = MultiModalConfig(mm_processor_cache_gb=mm_cache_gb) - # Minimal multimodal_config to satisfy references in - # Processor.process_inputs. - class _MockMMConfig: - def __init__(self, gb: float): - self.mm_processor_cache_gb = gb - - model_config.multimodal_config = _MockMMConfig(mm_cache_gb) # type: ignore[attr-defined] vllm_config = VllmConfig( - model_config=model_config, cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching), device_config=DeviceConfig(device="cpu"), ) - return InputProcessor(vllm_config, tokenizer=None) + return InputProcessor(vllm_config, renderer=TerratorchRenderer(model_config, {})) def test_multi_modal_uuids_length_mismatch_raises(monkeypatch): diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index d94951a0cffc..e786db52e2a1 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -11,6 +11,7 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput from vllm.plugins.io_processors import IOProcessor from vllm.pooling_params import PoolingParams +from vllm.renderers import RendererLike from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask from vllm.tokenizers import TokenizerLike @@ -26,6 +27,10 @@ class EngineClient(ABC): input_processor: InputProcessor io_processor: IOProcessor | None + @property + @abstractmethod + def renderer(self) -> RendererLike: ... + @property @abstractmethod def is_running(self) -> bool: ... diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index aceaa8bd45b8..49b82bc521d1 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -2,22 +2,15 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio -import inspect import json +import warnings from abc import ABC, abstractmethod -from collections import Counter, defaultdict, deque +from collections import Counter, defaultdict from collections.abc import Awaitable, Callable, Iterable from functools import cached_property, lru_cache, partial from pathlib import Path -from typing import Any, Generic, Literal, TypeAlias, TypeVar, cast - -import jinja2 -import jinja2.ext -import jinja2.meta -import jinja2.nodes -import jinja2.parser -import jinja2.sandbox -import transformers.utils.chat_template_utils as hf_chat_utils +from typing import Generic, Literal, TypeAlias, TypeVar, cast + from openai.types.chat import ( ChatCompletionAssistantMessageParam, ChatCompletionContentPartImageParam, @@ -38,7 +31,6 @@ from openai_harmony import Message as OpenAIHarmonyMessage from PIL import Image from pydantic import BaseModel, ConfigDict, TypeAdapter -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast, ProcessorMixin # pydantic needs the TypedDict from typing_extensions from typing_extensions import Required, TypedDict @@ -49,14 +41,26 @@ from vllm.model_executor.models import SupportsMultiModal from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalDataDict, MultiModalUUIDDict from vllm.multimodal.utils import MEDIA_CONNECTOR_REGISTRY, MediaConnector -from vllm.tokenizers import MistralTokenizer, TokenizerLike -from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path -from vllm.transformers_utils.processor import cached_get_processor from vllm.utils import random_uuid -from vllm.utils.func_utils import supports_kw logger = init_logger(__name__) + +def __getattr__(name: str): + if name == "resolve_hf_chat_template": + from vllm.renderers.hf import resolve_chat_template + + warnings.warn( + "`vllm.entrypoints.chat_utils.resolve_hf_chat_template` has been moved to " + "`vllm.renderers.hf.resolve_chat_template`. " + "The old name will be removed in v0.14.", + DeprecationWarning, + stacklevel=2, + ) + + return resolve_chat_template + + MODALITY_PLACEHOLDERS_MAP = { "image": "<##IMAGE##>", "audio": "<##AUDIO##>", @@ -295,325 +299,8 @@ class ConversationMessage(TypedDict, total=False): # Passed in by user ChatTemplateContentFormatOption = Literal["auto", "string", "openai"] -# Used internally -_ChatTemplateContentFormat = Literal["string", "openai"] - - -def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool: - if isinstance(node, jinja2.nodes.Name): - return node.ctx == "load" and node.name == varname - - return False - - -def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool: - if isinstance(node, jinja2.nodes.Getitem): - return ( - _is_var_access(node.node, varname) - and isinstance(node.arg, jinja2.nodes.Const) - and node.arg.value == key - ) - - if isinstance(node, jinja2.nodes.Getattr): - return _is_var_access(node.node, varname) and node.attr == key - - return False - - -def _is_var_or_elems_access( - node: jinja2.nodes.Node, - varname: str, - key: str | None = None, -) -> bool: - if isinstance(node, jinja2.nodes.Filter): - return node.node is not None and _is_var_or_elems_access( - node.node, varname, key - ) - if isinstance(node, jinja2.nodes.Test): - return _is_var_or_elems_access(node.node, varname, key) - - if isinstance(node, jinja2.nodes.Getitem) and isinstance( - node.arg, jinja2.nodes.Slice - ): - return _is_var_or_elems_access(node.node, varname, key) - - return _is_attr_access(node, varname, key) if key else _is_var_access(node, varname) - - -def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str): - # Global variable that is implicitly defined at the root - yield root, varname - - # Iterative BFS - related_varnames = deque([varname]) - while related_varnames: - related_varname = related_varnames.popleft() - - for assign_ast in root.find_all(jinja2.nodes.Assign): - lhs = assign_ast.target - rhs = assign_ast.node - - if _is_var_or_elems_access(rhs, related_varname): - assert isinstance(lhs, jinja2.nodes.Name) - yield assign_ast, lhs.name - - # Avoid infinite looping for self-assignment - if lhs.name != related_varname: - related_varnames.append(lhs.name) - - -# NOTE: The proper way to handle this is to build a CFG so that we can handle -# the scope in which each variable is defined, but that is too complicated -def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node): - messages_varnames = [ - varname for _, varname in _iter_nodes_assign_var_or_elems(root, "messages") - ] - - # Search for {%- for message in messages -%} loops - for loop_ast in root.find_all(jinja2.nodes.For): - loop_iter = loop_ast.iter - loop_target = loop_ast.target - - for varname in messages_varnames: - if _is_var_or_elems_access(loop_iter, varname): - assert isinstance(loop_target, jinja2.nodes.Name) - yield loop_ast, loop_target.name - break - - -def _iter_nodes_assign_content_item(root: jinja2.nodes.Node): - message_varnames = [ - varname for _, varname in _iter_nodes_assign_messages_item(root) - ] - - # Search for {%- for content in message['content'] -%} loops - for loop_ast in root.find_all(jinja2.nodes.For): - loop_iter = loop_ast.iter - loop_target = loop_ast.target - - for varname in message_varnames: - if _is_var_or_elems_access(loop_iter, varname, "content"): - assert isinstance(loop_target, jinja2.nodes.Name) - yield loop_ast, loop_target.name - break - - -def _try_extract_ast(chat_template: str) -> jinja2.nodes.Template | None: - try: - jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template) - return jinja_compiled.environment.parse(chat_template) - except Exception: - logger.exception("Error when compiling Jinja template") - return None - - -@lru_cache(maxsize=32) -def _detect_content_format( - chat_template: str, - *, - default: _ChatTemplateContentFormat, -) -> _ChatTemplateContentFormat: - jinja_ast = _try_extract_ast(chat_template) - if jinja_ast is None: - return default - - try: - next(_iter_nodes_assign_content_item(jinja_ast)) - except StopIteration: - return "string" - except Exception: - logger.exception("Error when parsing AST of Jinja template") - return default - else: - return "openai" - - -def resolve_mistral_chat_template( - chat_template: str | None, - **kwargs: Any, -) -> str | None: - if chat_template is not None or kwargs.get("chat_template_kwargs") is not None: - raise ValueError( - "'chat_template' or 'chat_template_kwargs' cannot be overridden " - "for mistral tokenizer." - ) - - return None - - -_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]() -""" -Used in `_try_get_processor_chat_template` to avoid calling -`cached_get_processor` again if the processor fails to be loaded. - -This is needed because `lru_cache` does not cache when an exception happens. -""" - - -def _try_get_processor_chat_template( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, - model_config: ModelConfig, -) -> str | None: - cache_key = (tokenizer.name_or_path, model_config.trust_remote_code) - if cache_key in _PROCESSOR_CHAT_TEMPLATES: - return _PROCESSOR_CHAT_TEMPLATES[cache_key] - - try: - processor = cached_get_processor( - tokenizer.name_or_path, - processor_cls=( - PreTrainedTokenizer, - PreTrainedTokenizerFast, - ProcessorMixin, - ), - trust_remote_code=model_config.trust_remote_code, - ) - if ( - isinstance(processor, ProcessorMixin) - and hasattr(processor, "chat_template") - and (chat_template := processor.chat_template) is not None - ): - _PROCESSOR_CHAT_TEMPLATES[cache_key] = chat_template - return chat_template - except Exception: - logger.debug( - "Failed to load AutoProcessor chat template for %s", - tokenizer.name_or_path, - exc_info=True, - ) - - _PROCESSOR_CHAT_TEMPLATES[cache_key] = None - return None - - -def resolve_hf_chat_template( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, - chat_template: str | None, - tools: list[dict[str, Any]] | None, - *, - model_config: ModelConfig, -) -> str | None: - # 1st priority: The given chat template - if chat_template is not None: - return chat_template - - # 2nd priority: AutoProcessor chat template, unless tool calling is enabled - if tools is None: - chat_template = _try_get_processor_chat_template(tokenizer, model_config) - if chat_template is not None: - return chat_template - - # 3rd priority: AutoTokenizer chat template - try: - return tokenizer.get_chat_template(chat_template, tools=tools) - except Exception: - logger.debug( - "Failed to load AutoTokenizer chat template for %s", - tokenizer.name_or_path, - exc_info=True, - ) - - # 4th priority: Predefined fallbacks - path = get_chat_template_fallback_path( - model_type=model_config.hf_config.model_type, - tokenizer_name_or_path=model_config.tokenizer, - ) - if path is not None: - logger.info_once( - "Loading chat template fallback for %s as there isn't one " - "defined on HF Hub.", - tokenizer.name_or_path, - ) - chat_template = load_chat_template(path) - else: - logger.debug_once( - "There is no chat template fallback for %s", tokenizer.name_or_path - ) - - return chat_template - - -def _resolve_chat_template_content_format( - chat_template: str | None, - tools: list[dict[str, Any]] | None, - tokenizer: TokenizerLike | None, - *, - model_config: ModelConfig, -) -> _ChatTemplateContentFormat: - if isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)): - hf_chat_template = resolve_hf_chat_template( - tokenizer, - chat_template=chat_template, - tools=tools, - model_config=model_config, - ) - else: - hf_chat_template = None - - jinja_text = ( - hf_chat_template - if isinstance(hf_chat_template, str) - else load_chat_template(chat_template, is_literal=True) - ) - - detected_format = ( - "string" - if jinja_text is None - else _detect_content_format(jinja_text, default="string") - ) - - return detected_format - - -@lru_cache -def _log_chat_template_content_format( - chat_template: str | None, - given_format: ChatTemplateContentFormatOption, - detected_format: ChatTemplateContentFormatOption, -): - logger.info( - "Detected the chat template content format to be '%s'. " - "You can set `--chat-template-content-format` to override this.", - detected_format, - ) - - if given_format != "auto" and given_format != detected_format: - logger.warning( - "You specified `--chat-template-content-format %s` " - "which is different from the detected format '%s'. " - "If our automatic detection is incorrect, please consider " - "opening a GitHub issue so that we can improve it: " - "https://github.com/vllm-project/vllm/issues/new/choose", - given_format, - detected_format, - ) - - -def resolve_chat_template_content_format( - chat_template: str | None, - tools: list[dict[str, Any]] | None, - given_format: ChatTemplateContentFormatOption, - tokenizer: TokenizerLike | None, - *, - model_config: ModelConfig, -) -> _ChatTemplateContentFormat: - if given_format != "auto": - return given_format - - detected_format = _resolve_chat_template_content_format( - chat_template, - tools, - tokenizer, - model_config=model_config, - ) - - _log_chat_template_content_format( - chat_template, - given_format=given_format, - detected_format=detected_format, - ) - - return detected_format +# After resolving "auto" +ChatTemplateContentFormat = Literal["string", "openai"] ModalityStr = Literal["image", "audio", "video", "image_embeds", "audio_embeds"] @@ -1536,7 +1223,7 @@ def _parse_chat_message_content_part( def _parse_chat_message_content( message: ChatCompletionMessageParam, mm_tracker: BaseMultiModalItemTracker, - content_format: _ChatTemplateContentFormat, + content_format: ChatTemplateContentFormat, interleave_strings: bool, ) -> list[ConversationMessage]: role = message["role"] @@ -1605,7 +1292,7 @@ def _postprocess_messages(messages: list[ConversationMessage]) -> None: def parse_chat_messages( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, - content_format: _ChatTemplateContentFormat, + content_format: ChatTemplateContentFormat, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -1636,7 +1323,7 @@ def parse_chat_messages( def parse_chat_messages_futures( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, - content_format: _ChatTemplateContentFormat, + content_format: ChatTemplateContentFormat, ) -> tuple[ list[ConversationMessage], Awaitable[MultiModalDataDict | None], @@ -1664,173 +1351,6 @@ def parse_chat_messages_futures( return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids() -# adapted from https://github.com/huggingface/transformers/blob/v4.56.2/src/transformers/utils/chat_template_utils.py#L398-L412 -# only preserve the parse function used to resolve chat template kwargs -class AssistantTracker(jinja2.ext.Extension): - tags = {"generation"} - - def parse(self, parser: jinja2.parser.Parser) -> jinja2.nodes.CallBlock: - lineno = next(parser.stream).lineno - body = parser.parse_statements(["name:endgeneration"], drop_needle=True) - call = self.call_method("_generation_support") - call_block = jinja2.nodes.CallBlock(call, [], [], body) - return call_block.set_lineno(lineno) - - -def _resolve_chat_template_kwargs( - chat_template: str, -): - env = jinja2.sandbox.ImmutableSandboxedEnvironment( - trim_blocks=True, - lstrip_blocks=True, - extensions=[AssistantTracker, jinja2.ext.loopcontrols], - ) - parsed_content = env.parse(chat_template) - template_vars = jinja2.meta.find_undeclared_variables(parsed_content) - return template_vars - - -_cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs) - - -@lru_cache -def _get_hf_base_chat_template_params() -> frozenset[str]: - # Get standard parameters from HuggingFace's base tokenizer class. - # This dynamically extracts parameters from PreTrainedTokenizer's - # apply_chat_template method, ensuring compatibility with tokenizers - # that use **kwargs to receive standard parameters. - - # Read signature from HF's base class - the single source of truth - base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template) - # Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders - return frozenset( - p.name - for p in base_sig.parameters.values() - if p.kind - not in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL) - ) - - -def resolve_chat_template_kwargs( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, - chat_template: str, - chat_template_kwargs: dict[str, Any], - raise_on_unexpected: bool = True, -) -> dict[str, Any]: - # We exclude chat_template from kwargs here, because - # chat template has been already resolved at this stage - unexpected_vars = {"chat_template", "tokenize"} - if raise_on_unexpected and ( - unexpected_in_kwargs := unexpected_vars & chat_template_kwargs.keys() - ): - raise ValueError( - "Found unexpected chat template kwargs from request: " - f"{unexpected_in_kwargs}" - ) - - fn_kw = { - k - for k in chat_template_kwargs - if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False) - } - template_vars = _cached_resolve_chat_template_kwargs(chat_template) - - # Allow standard HF parameters even if tokenizer uses **kwargs to receive them - hf_base_params = _get_hf_base_chat_template_params() - - accept_vars = (fn_kw | template_vars | hf_base_params) - unexpected_vars - return {k: v for k, v in chat_template_kwargs.items() if k in accept_vars} - - -def apply_hf_chat_template( - tokenizer: PreTrainedTokenizer | PreTrainedTokenizerFast, - conversation: list[ConversationMessage], - chat_template: str | None, - tools: list[dict[str, Any]] | None, - *, - model_config: ModelConfig, - **kwargs: Any, -) -> str: - hf_chat_template = resolve_hf_chat_template( - tokenizer, - chat_template=chat_template, - tools=tools, - model_config=model_config, - ) - - if hf_chat_template is None: - raise ValueError( - "As of transformers v4.44, default chat template is no longer " - "allowed, so you must provide a chat template if the tokenizer " - "does not define one." - ) - - resolved_kwargs = resolve_chat_template_kwargs( - tokenizer=tokenizer, - chat_template=hf_chat_template, - chat_template_kwargs=kwargs, - ) - - try: - return tokenizer.apply_chat_template( - conversation=conversation, # type: ignore[arg-type] - tools=tools, # type: ignore[arg-type] - chat_template=hf_chat_template, - tokenize=False, - **resolved_kwargs, - ) - - # External library exceptions can sometimes occur despite the framework's - # internal exception management capabilities. - except Exception as e: - # Log and report any library-related exceptions for further - # investigation. - logger.exception( - "An error occurred in `transformers` while applying chat template" - ) - raise ValueError(str(e)) from e - - -def apply_mistral_chat_template( - tokenizer: MistralTokenizer, - messages: list[ChatCompletionMessageParam], - chat_template: str | None, - tools: list[dict[str, Any]] | None, - **kwargs: Any, -) -> list[int]: - from mistral_common.exceptions import MistralCommonException - - # The return value of resolve_mistral_chat_template is always None, - # and we won't use it. - resolve_mistral_chat_template( - chat_template=chat_template, - **kwargs, - ) - - try: - return tokenizer.apply_chat_template( - messages=messages, - tools=tools, - **kwargs, - ) - # mistral-common uses assert statements to stop processing of input - # if input does not comply with the expected format. - # We convert those assertion errors to ValueErrors so they can be - # properly caught in the preprocessing_input step - except (AssertionError, MistralCommonException) as e: - raise ValueError(str(e)) from e - - # External library exceptions can sometimes occur despite the framework's - # internal exception management capabilities. - except Exception as e: - # Log and report any library-related exceptions for further - # investigation. - logger.exception( - "An error occurred in `mistral_common` while applying chat template" - ) - raise ValueError(str(e)) from e - - def get_history_tool_calls_cnt(conversation: list[ConversationMessage]): idx = 0 for msg in conversation: diff --git a/vllm/entrypoints/context.py b/vllm/entrypoints/context.py index a484a437c853..25aca81d619b 100644 --- a/vllm/entrypoints/context.py +++ b/vllm/entrypoints/context.py @@ -39,8 +39,8 @@ from vllm.entrypoints.tool_server import ToolServer from vllm.outputs import RequestOutput from vllm.reasoning.abs_reasoning_parsers import ReasoningParser -from vllm.tokenizers.protocol import TokenizerLike -from vllm.transformers_utils.tokenizer import AnyTokenizer +from vllm.renderers import RendererLike +from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid if TYPE_CHECKING: @@ -228,8 +228,8 @@ def __init__( self, *, response_messages: list[ResponseInputOutputItem], - tokenizer: AnyTokenizer, - reasoning_parser_cls: Callable[[AnyTokenizer], ReasoningParser] | None, + renderer: RendererLike, + reasoning_parser_cls: Callable[[TokenizerLike], ReasoningParser] | None, request: ResponsesRequest, available_tools: list[str] | None, tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None, @@ -248,7 +248,7 @@ def __init__( raise ValueError("reasoning_parser_cls must be provided.") self.parser = get_responses_parser_for_simple_context( - tokenizer=tokenizer, + tokenizer=renderer.get_tokenizer(), reasoning_parser_cls=reasoning_parser_cls, response_messages=response_messages, request=request, @@ -256,7 +256,8 @@ def __init__( ) self.tool_parser_cls = tool_parser_cls self.request = request - self.tokenizer = tokenizer + self.renderer = renderer + self.tokenizer = renderer.get_tokenizer() self.available_tools = available_tools or [] self._tool_sessions: dict[str, ClientSession | Tool] = {} diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 913324fd5f9c..7725fae16dd9 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -35,10 +35,6 @@ from vllm.entrypoints.chat_utils import ( ChatCompletionMessageParam, ChatTemplateContentFormatOption, - apply_hf_chat_template, - apply_mistral_chat_template, - parse_chat_messages, - resolve_chat_template_content_format, ) from vllm.entrypoints.score_utils import ( ScoreContentPartParam, @@ -71,7 +67,7 @@ from vllm.pooling_params import PoolingParams from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams from vllm.tasks import PoolingTask -from vllm.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.tokenizers.hf import get_cached_tokenizer from vllm.usage.usage_lib import UsageContext from vllm.utils.collection_utils import as_iter, is_list_of @@ -786,7 +782,7 @@ def preprocess_chat( tools: list[dict[str, Any]] | None = None, chat_template_kwargs: dict[str, Any] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, - ) -> list[TokensPrompt]: + ) -> list[TextPrompt | TokensPrompt]: """ Generate prompt for a chat conversation. The pre-processed prompt can then be used as input for the other LLM methods. @@ -807,63 +803,27 @@ def preprocess_chat( # messages is list[...] list_of_messages = [cast(list[ChatCompletionMessageParam], messages)] - tokenizer = self.get_tokenizer() - model_config = self.model_config - resolved_content_format = resolve_chat_template_content_format( - chat_template, - tools, - chat_template_content_format, - tokenizer, - model_config=model_config, - ) + renderer = self.llm_engine.renderer - _chat_template_kwargs: dict[str, Any] = dict( - chat_template=chat_template, - add_generation_prompt=add_generation_prompt, - continue_final_message=continue_final_message, - tools=tools, - ) - _chat_template_kwargs.update(chat_template_kwargs or {}) + chat_template_kwargs = { + "chat_template": chat_template, + "add_generation_prompt": add_generation_prompt, + "continue_final_message": continue_final_message, + "tools": tools, + **(chat_template_kwargs or {}), + } - prompts: list[TokensPrompt] = [] + prompts = list[TextPrompt | TokensPrompt]() for msgs in list_of_messages: - # NOTE: _parse_chat_message_content_parts() currently doesn't + # NOTE: parse_mesrender_messagessages() currently doesn't # handle mm_processor_kwargs, since there is no implementation in # the chat message parsing for it. - conversation, mm_data, mm_uuids = parse_chat_messages( + _, prompt = renderer.render_messages( msgs, - model_config, - content_format=resolved_content_format, + chat_template_content_format=chat_template_content_format, + **chat_template_kwargs, ) - - if isinstance(tokenizer, MistralTokenizer): - prompt_token_ids = apply_mistral_chat_template( - tokenizer, - messages=msgs, - **_chat_template_kwargs, - ) - else: - prompt_str = apply_hf_chat_template( - tokenizer=tokenizer, - conversation=conversation, - model_config=model_config, - **_chat_template_kwargs, - ) - # Special tokens are already included in chat templates so - # should not be added by the tokenizer in this case. - prompt_token_ids = tokenizer.encode( - prompt_str, add_special_tokens=False - ) - - prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) - - if mm_data is not None: - prompt["multi_modal_data"] = mm_data - - if mm_uuids is not None: - prompt["multi_modal_uuids"] = mm_uuids - if mm_processor_kwargs is not None: prompt["mm_processor_kwargs"] = mm_processor_kwargs @@ -1292,9 +1252,6 @@ def _cross_encoding_score( ) -> list[ScoringRequestOutput]: model_config = self.model_config - if isinstance(tokenizer, MistralTokenizer): - raise ValueError("Score API is not supported for Mistral tokenizer") - if len(data_1) == 1: data_1 = data_1 * len(data_2) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index c6333d170c66..1476aa8f38ce 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -188,7 +188,8 @@ async def create_chat_completion( model_name = self.models.model_name(lora_request) - tokenizer = await self.engine_client.get_tokenizer() + renderer = self.engine_client.renderer + tokenizer = renderer.get_tokenizer() tool_parser = self.tool_parser @@ -236,7 +237,7 @@ async def create_chat_completion( engine_prompts, ) = await self._preprocess_chat( request, - tokenizer, + renderer, request.messages, chat_template=request.chat_template or self.chat_template, chat_template_content_format=self.chat_template_content_format, diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 99936f588f28..2bc89c4d0d24 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -6,7 +6,6 @@ import time import traceback from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence -from concurrent.futures import ThreadPoolExecutor from dataclasses import dataclass, field from http import HTTPStatus from typing import Any, ClassVar, Generic, TypeAlias, TypeVar @@ -49,7 +48,6 @@ ScoreRequest, ScoreResponse, ) -from vllm.transformers_utils.tokenizer import AnyTokenizer if sys.version_info >= (3, 12): from typing import TypedDict @@ -67,10 +65,6 @@ ChatCompletionMessageParam, ChatTemplateContentFormatOption, ConversationMessage, - apply_hf_chat_template, - apply_mistral_chat_template, - parse_chat_messages_futures, - resolve_chat_template_content_format, ) from vllm.entrypoints.context import ConversationContext from vllm.entrypoints.logger import RequestLogger @@ -99,7 +93,7 @@ ) from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse from vllm.entrypoints.utils import _validate_truncation_size -from vllm.inputs.data import PromptType +from vllm.inputs.data import PromptType, SingletonPrompt from vllm.inputs.data import TokensPrompt as EngineTokensPrompt from vllm.inputs.parse import ( PromptComponents, @@ -109,15 +103,13 @@ from vllm.logger import init_logger from vllm.logprobs import Logprob, PromptLogprobs from vllm.lora.request import LoRARequest -from vllm.multimodal import ( # noqa: F401 - Required to resolve Pydantic error in RequestProcessingMixin - MultiModalDataDict, - MultiModalUUIDDict, -) +from vllm.multimodal import MultiModalDataDict from vllm.outputs import CompletionOutput, PoolingRequestOutput, RequestOutput from vllm.pooling_params import PoolingParams from vllm.reasoning import ReasoningParser, ReasoningParserManager +from vllm.renderers import RendererLike from vllm.sampling_params import BeamSearchParams, SamplingParams -from vllm.tokenizers import DeepseekV32Tokenizer, MistralTokenizer, TokenizerLike +from vllm.tokenizers import TokenizerLike from vllm.tracing import ( contains_trace_headers, extract_trace_headers, @@ -127,10 +119,8 @@ from vllm.utils.async_utils import ( AsyncMicrobatchTokenizer, collect_from_async_generator, - make_async, merge_async_iterators, ) -from vllm.utils.collection_utils import is_list_of from vllm.v1.engine import EngineCoreRequest logger = init_logger(__name__) @@ -183,7 +173,7 @@ class EmbedsPrompt(TypedDict): prompt_embeds: torch.Tensor -RequestPrompt: TypeAlias = list[int] | str | TextTokensPrompt | EmbedsPrompt +RequestPrompt: TypeAlias = list[int] | TextTokensPrompt | EmbedsPrompt | SingletonPrompt def is_text_tokens_prompt(prompt: RequestPrompt) -> TypeIs[TextTokensPrompt]: @@ -235,7 +225,6 @@ class ResponseGenerationMixin: @dataclass(kw_only=True) class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[RequestT]): - # Shared across all requests request: RequestT raw_request: Request | None = None model_name: str @@ -243,9 +232,6 @@ class ServeContext(RequestProcessingMixin, ResponseGenerationMixin, Generic[Requ created_time: int = field(default_factory=lambda: int(time.time())) lora_request: LoRARequest | None = None - # Shared across most requests - tokenizer: TokenizerLike | None = None - @dataclass(kw_only=True) class ClassificationServeContext(ServeContext[ClassificationRequest]): @@ -281,16 +267,13 @@ def __init__( self.request_logger = request_logger self.return_tokens_as_token_ids = return_tokens_as_token_ids - self._tokenizer_executor = ThreadPoolExecutor(max_workers=1) - self._apply_mistral_chat_template_async = make_async( - apply_mistral_chat_template, executor=self._tokenizer_executor - ) self._async_tokenizer_pool: dict[TokenizerLike, AsyncMicrobatchTokenizer] = {} self.log_error_stack = log_error_stack self.input_processor = self.models.input_processor self.io_processor = self.models.io_processor + self.renderer = self.models.renderer self.model_config = self.models.model_config self.max_model_len = self.model_config.max_model_len @@ -1084,7 +1067,7 @@ def _validate_chat_template( async def _preprocess_chat( self, request: ChatLikeRequest | ResponsesRequest, - tokenizer: TokenizerLike | None, + renderer: RendererLike, messages: list[ChatCompletionMessageParam], chat_template: str | None, chat_template_content_format: ChatTemplateContentFormatOption, @@ -1100,56 +1083,33 @@ async def _preprocess_chat( Sequence[RequestPrompt], list[EngineTokensPrompt], ]: - model_config = self.model_config - - resolved_content_format = resolve_chat_template_content_format( - chat_template, - tool_dicts, - chat_template_content_format, - tokenizer, - model_config=model_config, - ) - conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + chat_template_kwargs = { + "chat_template": chat_template, + "add_generation_prompt": add_generation_prompt, + "continue_final_message": continue_final_message, + "tools": tool_dicts, + "documents": documents, + **(chat_template_kwargs or {}), + } + + conversation, engine_prompt = await renderer.render_messages_async( messages, - model_config, - content_format=resolved_content_format, - ) - - _chat_template_kwargs: dict[str, Any] = dict( - chat_template=chat_template, - add_generation_prompt=add_generation_prompt, - continue_final_message=continue_final_message, - tools=tool_dicts, - documents=documents, + chat_template_content_format=chat_template_content_format, + **chat_template_kwargs, ) - _chat_template_kwargs.update(chat_template_kwargs or {}) - - request_prompt: str | list[int] - if tokenizer is None: - request_prompt = "placeholder" - elif isinstance(tokenizer, MistralTokenizer): - request_prompt = await self._apply_mistral_chat_template_async( - tokenizer, - messages=messages, - **_chat_template_kwargs, - ) - elif isinstance(tokenizer, DeepseekV32Tokenizer): - request_prompt = tokenizer.apply_chat_template( - conversation=conversation, - messages=messages, - model_config=model_config, - **_chat_template_kwargs, - ) - else: - request_prompt = apply_hf_chat_template( - tokenizer=tokenizer, - conversation=conversation, - model_config=model_config, - **_chat_template_kwargs, + if "prompt_token_ids" not in engine_prompt: + engine_prompt = await self._tokenize_prompt_input_async( + request, + renderer.get_tokenizer(), + engine_prompt["prompt"], + add_special_tokens=add_special_tokens, ) - mm_data = await mm_data_future + if request.mm_processor_kwargs is not None: + engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs + if (cache_salt := getattr(request, "cache_salt", None)) is not None: + engine_prompt["cache_salt"] = cache_salt # tool parsing is done only if a tool_parser has been set and if # tool_choice is not "none" (if tool_choice is "none" but a tool_parser @@ -1165,49 +1125,11 @@ async def _preprocess_chat( "or Responses API requests." ) raise NotImplementedError(msg) - request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore - - if tokenizer is None: - assert isinstance(request_prompt, str), ( - "Prompt has to be a string", - "when the tokenizer is not initialised", - ) - prompt_inputs = TextTokensPrompt( - prompt=request_prompt, prompt_token_ids=[1] - ) - elif isinstance(request_prompt, str): - prompt_inputs = await self._tokenize_prompt_input_async( - request, - tokenizer, - request_prompt, - add_special_tokens=add_special_tokens, - ) - else: - # For MistralTokenizer - assert is_list_of(request_prompt, int), ( - "Prompt has to be either a string or a list of token ids" - ) - prompt_inputs = TextTokensPrompt( - prompt=tokenizer.decode(request_prompt), - prompt_token_ids=request_prompt, - ) - - engine_prompt = EngineTokensPrompt( - prompt_token_ids=prompt_inputs["prompt_token_ids"] - ) - if mm_data is not None: - engine_prompt["multi_modal_data"] = mm_data - - if mm_uuids is not None: - engine_prompt["multi_modal_uuids"] = mm_uuids - if request.mm_processor_kwargs is not None: - engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs - - if hasattr(request, "cache_salt") and request.cache_salt is not None: - engine_prompt["cache_salt"] = request.cache_salt + tokenizer = renderer.get_tokenizer() + request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore - return conversation, [request_prompt], [engine_prompt] + return conversation, [engine_prompt], [engine_prompt] async def _process_inputs( self, @@ -1239,7 +1161,7 @@ async def _process_inputs( async def _render_next_turn( self, request: ResponsesRequest, - tokenizer: AnyTokenizer, + renderer: RendererLike, messages: list[ResponseInputOutputItem], tool_dicts: list[dict[str, Any]] | None, tool_parser, @@ -1252,7 +1174,7 @@ async def _render_next_turn( _, request_prompts, engine_prompts = await self._preprocess_chat( request, - tokenizer, + renderer, new_messages, tool_dicts=tool_dicts, tool_parser=tool_parser, @@ -1330,7 +1252,7 @@ async def _generate_with_builtin_tools( elif isinstance(context, ParsableContext): request_prompts, engine_prompts = await self._render_next_turn( context.request, - context.tokenizer, + context.renderer, context.parser.response_messages, context.tool_dicts, context.tool_parser_cls, diff --git a/vllm/entrypoints/openai/serving_models.py b/vllm/entrypoints/openai/serving_models.py index 953398a9a72a..d4335016f635 100644 --- a/vllm/entrypoints/openai/serving_models.py +++ b/vllm/entrypoints/openai/serving_models.py @@ -71,6 +71,7 @@ def __init__( self.input_processor = self.engine_client.input_processor self.io_processor = self.engine_client.io_processor + self.renderer = self.engine_client.renderer self.model_config = self.engine_client.model_config self.max_model_len = self.model_config.max_model_len diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index 91616a78e11d..1bb0c76f8e8e 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -108,6 +108,7 @@ from vllm.logprobs import Logprob as SampleLogprob from vllm.logprobs import SampleLogprobs from vllm.outputs import CompletionOutput +from vllm.renderers import RendererLike from vllm.sampling_params import SamplingParams, StructuredOutputsParams from vllm.tokenizers import TokenizerLike from vllm.utils import random_uuid @@ -346,7 +347,8 @@ async def create_responses( try: lora_request = self._maybe_get_adapters(request) model_name = self.models.model_name(lora_request) - tokenizer = await self.engine_client.get_tokenizer() + renderer = self.engine_client.renderer + tokenizer = renderer.get_tokenizer() if self.use_harmony: messages, request_prompts, engine_prompts = ( @@ -354,7 +356,7 @@ async def create_responses( ) else: messages, request_prompts, engine_prompts = await self._make_request( - request, prev_response, tokenizer + request, prev_response, renderer ) except ( @@ -420,7 +422,7 @@ async def create_responses( # tokens during generation instead of at the end context = ParsableContext( response_messages=messages, - tokenizer=tokenizer, + renderer=renderer, reasoning_parser_cls=self.reasoning_parser, request=request, tool_parser_cls=self.tool_parser, @@ -548,7 +550,7 @@ async def _make_request( self, request: ResponsesRequest, prev_response: ResponsesResponse | None, - tokenizer: TokenizerLike, + renderer: RendererLike, ): tool_dicts = construct_tool_dicts(request.tools, request.tool_choice) # Construct the input messages. @@ -560,7 +562,7 @@ async def _make_request( ) _, request_prompts, engine_prompts = await self._preprocess_chat( request, - tokenizer, + renderer, messages, tool_dicts=tool_dicts, tool_parser=self.tool_parser, diff --git a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py index 19c1c83268ed..14cf2f38b70c 100644 --- a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py @@ -22,7 +22,8 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.mistral import MistralTokenizer logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py index 4655da8dd454..92b09917c252 100644 --- a/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/jamba_tool_parser.py @@ -21,7 +21,8 @@ from vllm.entrypoints.openai.tool_parsers import ToolParser from vllm.entrypoints.openai.tool_parsers.utils import extract_intermediate_diff from vllm.logger import init_logger -from vllm.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.mistral import MistralTokenizer logger = init_logger(__name__) diff --git a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py index aa5089ffe84d..54a90cb0f7b0 100644 --- a/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py +++ b/vllm/entrypoints/openai/tool_parsers/mistral_tool_parser.py @@ -6,6 +6,7 @@ from enum import Enum, auto from random import choices from string import ascii_letters, digits +from typing import Any import ijson import regex as re @@ -24,7 +25,8 @@ ToolParser, ) from vllm.logger import init_logger -from vllm.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.mistral import MistralTokenizer logger = init_logger(__name__) @@ -112,6 +114,8 @@ def __init__(self, tokenizer: TokenizerLike): "the tokenizer!" ) + self.prev_tool_call_arr: list[dict[str, Any]] + def adjust_request(self, request: ChatCompletionRequest) -> ChatCompletionRequest: request = super().adjust_request(request) if ( diff --git a/vllm/entrypoints/pooling/classify/serving.py b/vllm/entrypoints/pooling/classify/serving.py index d6d3825daf7b..377d037b2c10 100644 --- a/vllm/entrypoints/pooling/classify/serving.py +++ b/vllm/entrypoints/pooling/classify/serving.py @@ -52,8 +52,6 @@ async def _preprocess( """ ctx = cast(ClassificationServeContext, ctx) try: - ctx.tokenizer = await self.engine_client.get_tokenizer() - request_obj = ctx.request if isinstance(request_obj, ClassificationChatRequest): @@ -78,7 +76,7 @@ async def _preprocess( engine_prompts, ) = await self._preprocess_chat( cast(ChatCompletionRequest, chat_request), - ctx.tokenizer, + self.renderer, messages, chat_template=( chat_request.chat_template @@ -106,7 +104,7 @@ async def _preprocess( ctx.engine_prompts = [] return None - renderer = self._get_renderer(ctx.tokenizer) + renderer = self._get_renderer(self.renderer.tokenizer) prompt_input = cast(str | list[str], input_data) ctx.engine_prompts = await renderer.render_prompt( prompt_or_prompts=prompt_input, diff --git a/vllm/entrypoints/pooling/embed/serving.py b/vllm/entrypoints/pooling/embed/serving.py index 868a3cb017a6..6675b1fc8793 100644 --- a/vllm/entrypoints/pooling/embed/serving.py +++ b/vllm/entrypoints/pooling/embed/serving.py @@ -79,9 +79,6 @@ async def _preprocess( try: ctx.lora_request = self._maybe_get_adapters(ctx.request) - tokenizer = await self.engine_client.get_tokenizer() - renderer = self._get_renderer(tokenizer) - if isinstance(ctx.request, EmbeddingChatRequest): ( _, @@ -89,7 +86,7 @@ async def _preprocess( ctx.engine_prompts, ) = await self._preprocess_chat( ctx.request, - tokenizer, + self.renderer, ctx.request.messages, chat_template=ctx.request.chat_template or ctx.chat_template, chat_template_content_format=ctx.chat_template_content_format, @@ -98,6 +95,8 @@ async def _preprocess( add_special_tokens=ctx.request.add_special_tokens, ) else: + tokenizer = await self.engine_client.get_tokenizer() + renderer = self._get_renderer(tokenizer) ctx.engine_prompts = await renderer.render_prompt( prompt_or_prompts=ctx.request.input, config=self._build_render_config(ctx.request), diff --git a/vllm/entrypoints/pooling/pooling/serving.py b/vllm/entrypoints/pooling/pooling/serving.py index 7fb767e26d01..d256d4c2178f 100644 --- a/vllm/entrypoints/pooling/pooling/serving.py +++ b/vllm/entrypoints/pooling/pooling/serving.py @@ -94,12 +94,6 @@ async def create_pooling( try: lora_request = self._maybe_get_adapters(request) - if self.model_config.skip_tokenizer_init: - tokenizer = None - else: - tokenizer = await self.engine_client.get_tokenizer() - renderer = self._get_renderer(tokenizer) - if getattr(request, "dimensions", None) is not None: return self.create_error_response( "dimensions is currently not supported" @@ -143,7 +137,7 @@ async def create_pooling( engine_prompts, ) = await self._preprocess_chat( request, - tokenizer, + self.renderer, request.messages, chat_template=request.chat_template or self.chat_template, chat_template_content_format=self.chat_template_content_format, @@ -154,6 +148,11 @@ async def create_pooling( add_special_tokens=request.add_special_tokens, ) elif isinstance(request, PoolingCompletionRequest): + if self.model_config.skip_tokenizer_init: + tokenizer = None + else: + tokenizer = await self.engine_client.get_tokenizer() + renderer = self._get_renderer(tokenizer) engine_prompts = await renderer.render_prompt( prompt_or_prompts=request.input, config=self._build_render_config(request), diff --git a/vllm/entrypoints/pooling/score/serving.py b/vllm/entrypoints/pooling/score/serving.py index e5a66783005a..846944948083 100644 --- a/vllm/entrypoints/pooling/score/serving.py +++ b/vllm/entrypoints/pooling/score/serving.py @@ -3,6 +3,7 @@ import asyncio import time from collections.abc import AsyncGenerator, Mapping +from concurrent.futures import ThreadPoolExecutor from typing import Any from fastapi import Request @@ -38,7 +39,8 @@ from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.outputs import PoolingRequestOutput, ScoringRequestOutput -from vllm.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.async_utils import make_async, merge_async_iterators logger = init_logger(__name__) @@ -60,6 +62,8 @@ def __init__( log_error_stack=log_error_stack, ) + self._tokenizer_executor = ThreadPoolExecutor(max_workers=1) + async def _embedding_score( self, tokenizer: TokenizerLike, diff --git a/vllm/entrypoints/serve/tokenize/serving.py b/vllm/entrypoints/serve/tokenize/serving.py index 979da02d1450..a1293c82c355 100644 --- a/vllm/entrypoints/serve/tokenize/serving.py +++ b/vllm/entrypoints/serve/tokenize/serving.py @@ -64,9 +64,6 @@ async def create_tokenize( try: lora_request = self._maybe_get_adapters(request) - tokenizer = await self.engine_client.get_tokenizer() - renderer = self._get_renderer(tokenizer) - if isinstance(request, TokenizeChatRequest): tool_dicts = ( None @@ -86,7 +83,7 @@ async def create_tokenize( engine_prompts, ) = await self._preprocess_chat( request, - tokenizer, + self.renderer, request.messages, tool_dicts=tool_dicts, chat_template=request.chat_template or self.chat_template, @@ -97,6 +94,8 @@ async def create_tokenize( add_special_tokens=request.add_special_tokens, ) else: + tokenizer = await self.engine_client.get_tokenizer() + renderer = self._get_renderer(tokenizer) engine_prompts = await renderer.render_prompt( prompt_or_prompts=request.prompt, config=self._build_render_config(request), diff --git a/vllm/entrypoints/utils.py b/vllm/entrypoints/utils.py index daeeb995bc74..f4a633c69cb0 100644 --- a/vllm/entrypoints/utils.py +++ b/vllm/entrypoints/utils.py @@ -30,7 +30,7 @@ from vllm.entrypoints.openai.serving_models import LoRAModulePath from vllm.logger import init_logger from vllm.platforms import current_platform -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.argparse_utils import FlexibleArgumentParser logger = init_logger(__name__) diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py index 0372b06d0017..5c8388f5b4e8 100644 --- a/vllm/inputs/preprocess.py +++ b/vllm/inputs/preprocess.py @@ -17,6 +17,7 @@ MultiModalUUIDDict, ) from vllm.multimodal.processing import BaseMultiModalProcessor +from vllm.renderers import renderer_from_config from vllm.tokenizers import TokenizerLike from vllm.utils.jsontree import json_iter_leaves from vllm.v1.metrics.stats import MultiModalCacheStats @@ -46,26 +47,28 @@ class InputPreprocessor: def __init__( self, model_config: ModelConfig, - tokenizer: TokenizerLike | None, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, mm_processor_cache: BaseMultiModalProcessorCache | None = None, ) -> None: super().__init__() self.model_config = model_config - self.tokenizer = tokenizer + self.renderer = renderer_from_config(model_config) self.mm_registry = mm_registry self.mm_processor_cache = mm_processor_cache self.mm_cache_stats = MultiModalCacheStats() if mm_processor_cache else None - def get_tokenizer(self) -> TokenizerLike: - if self.tokenizer is None: - raise ValueError( - "You cannot pass text prompts when `skip_tokenizer_init=True`" - ) + @property + def tokenizer(self) -> TokenizerLike | None: + return self.renderer.tokenizer + + @tokenizer.setter + def tokenizer(self, tokenizer: TokenizerLike | None) -> None: + self.renderer.tokenizer = tokenizer - return self.tokenizer + def get_tokenizer(self) -> TokenizerLike: + return self.renderer.get_tokenizer() def get_bos_token_id(self) -> int | None: if self.tokenizer is None: diff --git a/vllm/model_executor/models/pixtral.py b/vllm/model_executor/models/pixtral.py index faf2d80d24bb..555e6ea4b8cb 100644 --- a/vllm/model_executor/models/pixtral.py +++ b/vllm/model_executor/models/pixtral.py @@ -59,7 +59,8 @@ from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors -from vllm.tokenizers import MistralTokenizer, cached_tokenizer_from_config +from vllm.tokenizers import cached_tokenizer_from_config +from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.tensor_schema import TensorSchema, TensorShape from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP diff --git a/vllm/model_executor/models/voxtral.py b/vllm/model_executor/models/voxtral.py index 7b408248ec74..331f0c54ecfb 100644 --- a/vllm/model_executor/models/voxtral.py +++ b/vllm/model_executor/models/voxtral.py @@ -51,7 +51,8 @@ ) from vllm.multimodal.profiling import BaseDummyInputsBuilder, ProcessorInputs from vllm.sequence import IntermediateTensors -from vllm.tokenizers import MistralTokenizer, cached_tokenizer_from_config +from vllm.tokenizers import cached_tokenizer_from_config +from vllm.tokenizers.mistral import MistralTokenizer from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsTranscription from .utils import init_vllm_registered_model, maybe_prefix diff --git a/vllm/reasoning/mistral_reasoning_parser.py b/vllm/reasoning/mistral_reasoning_parser.py index b61e50c188f8..1eb4ead69916 100644 --- a/vllm/reasoning/mistral_reasoning_parser.py +++ b/vllm/reasoning/mistral_reasoning_parser.py @@ -6,7 +6,7 @@ from vllm.logger import init_logger from vllm.reasoning import ReasoningParser from vllm.reasoning.deepseek_r1_reasoning_parser import DeepSeekR1ReasoningParser -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer logger = init_logger(__name__) diff --git a/vllm/renderers/__init__.py b/vllm/renderers/__init__.py new file mode 100644 index 000000000000..cd6a11dcc833 --- /dev/null +++ b/vllm/renderers/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from .protocol import RendererLike +from .registry import RendererRegistry, renderer_from_config + +__all__ = ["RendererLike", "RendererRegistry", "renderer_from_config"] diff --git a/vllm/renderers/deepseekv32.py b/vllm/renderers/deepseekv32.py new file mode 100644 index 000000000000..40878af9f262 --- /dev/null +++ b/vllm/renderers/deepseekv32.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +from vllm.config import ModelConfig +from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, + ConversationMessage, + parse_chat_messages, + parse_chat_messages_futures, +) +from vllm.inputs import TextPrompt, TokensPrompt +from vllm.logger import init_logger +from vllm.tokenizers import get_tokenizer +from vllm.tokenizers.deepseekv32 import DeepseekV32Tokenizer + +from .protocol import RendererLike + +logger = init_logger(__name__) + + +class DeepseekV32Renderer(RendererLike): + @classmethod + def from_config( + cls, + config: ModelConfig, + tokenizer_kwargs: dict[str, Any], + ) -> "RendererLike": + return cls(config, tokenizer_kwargs) + + def __init__( + self, + config: ModelConfig, + tokenizer_kwargs: dict[str, Any], + ) -> None: + super().__init__() + + self.config = config + + if config.skip_tokenizer_init: + tokenizer = None + else: + tokenizer = get_tokenizer(DeepseekV32Tokenizer, **tokenizer_kwargs) + + self._tokenizer = tokenizer + + @property + def tokenizer(self) -> DeepseekV32Tokenizer | None: + return self._tokenizer + + # NOTE: Remove this once LLM.tokenizer.setter is removed + @tokenizer.setter + def tokenizer(self, tokenizer: DeepseekV32Tokenizer | None) -> None: + self._tokenizer = tokenizer + + def get_tokenizer(self) -> DeepseekV32Tokenizer: + tokenizer = self.tokenizer + if tokenizer is None: + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") + + return tokenizer + + def render_messages( + self, + messages: list[ChatCompletionMessageParam], + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + tokenizer = self.get_tokenizer() + conversation, mm_data, mm_uuids = parse_chat_messages( + messages, + self.config, + content_format="string", + ) + + prompt_raw = tokenizer.apply_chat_template( + conversation=conversation, + messages=messages, + **kwargs, + ) + if isinstance(prompt_raw, str): + prompt = TextPrompt(prompt=prompt_raw) + else: + prompt = TokensPrompt(prompt_token_ids=prompt_raw) + + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt + + async def render_messages_async( + self, + messages: list[ChatCompletionMessageParam], + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + tokenizer = self.get_tokenizer() + conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + messages, + self.config, + content_format="string", + ) + + prompt_raw = tokenizer.apply_chat_template( + conversation=conversation, + messages=messages, + **kwargs, + ) + if isinstance(prompt_raw, str): + prompt = TextPrompt(prompt=prompt_raw) + else: + prompt = TokensPrompt(prompt_token_ids=prompt_raw) + + if mm_data_future is not None: + prompt["multi_modal_data"] = await mm_data_future + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py new file mode 100644 index 000000000000..dec21c031bee --- /dev/null +++ b/vllm/renderers/hf.py @@ -0,0 +1,596 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import inspect +from collections import deque +from collections.abc import Set +from functools import lru_cache +from typing import Any + +import jinja2 +import jinja2.ext +import jinja2.meta +import jinja2.nodes +import jinja2.parser +import jinja2.sandbox + +from vllm.config import ModelConfig +from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, + ChatTemplateContentFormat, + ChatTemplateContentFormatOption, + ConversationMessage, + load_chat_template, + parse_chat_messages, + parse_chat_messages_futures, +) +from vllm.inputs import TextPrompt, TokensPrompt +from vllm.logger import init_logger +from vllm.tokenizers import get_tokenizer +from vllm.tokenizers.hf import HfTokenizer +from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path +from vllm.transformers_utils.processor import cached_get_processor +from vllm.utils.func_utils import supports_kw + +from .protocol import RendererLike + +logger = init_logger(__name__) + + +_PROCESSOR_CHAT_TEMPLATES = dict[tuple[str, bool], str | None]() +""" +Used in `_try_get_processor_chat_template` to avoid calling +`cached_get_processor` again if the processor fails to be loaded. + +This is needed because `lru_cache` does not cache when an exception happens. +""" + + +def _try_get_processor_chat_template( + tokenizer: HfTokenizer, + *, + trust_remote_code: bool, +) -> str | None: + cache_key = (tokenizer.name_or_path, trust_remote_code) + if cache_key in _PROCESSOR_CHAT_TEMPLATES: + return _PROCESSOR_CHAT_TEMPLATES[cache_key] + + from transformers import ( + PreTrainedTokenizer, + PreTrainedTokenizerFast, + ProcessorMixin, + ) + + try: + processor = cached_get_processor( + tokenizer.name_or_path, + processor_cls=( + PreTrainedTokenizer, + PreTrainedTokenizerFast, + ProcessorMixin, + ), + trust_remote_code=trust_remote_code, + ) + if ( + isinstance(processor, ProcessorMixin) + and hasattr(processor, "chat_template") + and (chat_template := processor.chat_template) is not None + ): + _PROCESSOR_CHAT_TEMPLATES[cache_key] = chat_template + return chat_template + except Exception: + logger.debug( + "Failed to load AutoProcessor chat template for %s", + tokenizer.name_or_path, + exc_info=True, + ) + + _PROCESSOR_CHAT_TEMPLATES[cache_key] = None + return None + + +def resolve_chat_template( + tokenizer: HfTokenizer, + chat_template: str | None, + tools: list[dict[str, Any]] | None, + *, + model_config: "ModelConfig", +) -> str | None: + # 1st priority: The given chat template + if chat_template is not None: + return chat_template + + # 2nd priority: AutoProcessor chat template, unless tool calling is enabled + if tools is None: + chat_template = _try_get_processor_chat_template( + tokenizer, + trust_remote_code=model_config.trust_remote_code, + ) + if chat_template is not None: + return chat_template + + # 3rd priority: AutoTokenizer chat template + try: + return tokenizer.get_chat_template(chat_template, tools=tools) + except Exception: + logger.debug( + "Failed to load AutoTokenizer chat template for %s", + tokenizer.name_or_path, + exc_info=True, + ) + + # 4th priority: Predefined fallbacks + path = get_chat_template_fallback_path( + model_type=model_config.hf_config.model_type, + tokenizer_name_or_path=tokenizer.name_or_path, + ) + if path is not None: + logger.info_once( + "Loading chat template fallback for %s as there isn't one " + "defined on HF Hub.", + tokenizer.name_or_path, + ) + chat_template = load_chat_template(path) + else: + logger.debug_once( + "There is no chat template fallback for %s", tokenizer.name_or_path + ) + + return chat_template + + +def _is_var_access(node: jinja2.nodes.Node, varname: str) -> bool: + if isinstance(node, jinja2.nodes.Name): + return node.ctx == "load" and node.name == varname + + return False + + +def _is_attr_access(node: jinja2.nodes.Node, varname: str, key: str) -> bool: + if isinstance(node, jinja2.nodes.Getitem): + return ( + _is_var_access(node.node, varname) + and isinstance(node.arg, jinja2.nodes.Const) + and node.arg.value == key + ) + + if isinstance(node, jinja2.nodes.Getattr): + return _is_var_access(node.node, varname) and node.attr == key + + return False + + +def _is_var_or_elems_access( + node: jinja2.nodes.Node, + varname: str, + key: str | None = None, +) -> bool: + if isinstance(node, jinja2.nodes.Filter): + return node.node is not None and _is_var_or_elems_access( + node.node, varname, key + ) + if isinstance(node, jinja2.nodes.Test): + return _is_var_or_elems_access(node.node, varname, key) + + if isinstance(node, jinja2.nodes.Getitem) and isinstance( + node.arg, jinja2.nodes.Slice + ): + return _is_var_or_elems_access(node.node, varname, key) + + return _is_attr_access(node, varname, key) if key else _is_var_access(node, varname) + + +def _iter_nodes_assign_var_or_elems(root: jinja2.nodes.Node, varname: str): + # Global variable that is implicitly defined at the root + yield root, varname + + # Iterative BFS + related_varnames = deque([varname]) + while related_varnames: + related_varname = related_varnames.popleft() + + for assign_ast in root.find_all(jinja2.nodes.Assign): + lhs = assign_ast.target + rhs = assign_ast.node + + if _is_var_or_elems_access(rhs, related_varname): + assert isinstance(lhs, jinja2.nodes.Name) + yield assign_ast, lhs.name + + # Avoid infinite looping for self-assignment + if lhs.name != related_varname: + related_varnames.append(lhs.name) + + +# NOTE: The proper way to handle this is to build a CFG so that we can handle +# the scope in which each variable is defined, but that is too complicated +def _iter_nodes_assign_messages_item(root: jinja2.nodes.Node): + messages_varnames = [ + varname for _, varname in _iter_nodes_assign_var_or_elems(root, "messages") + ] + + # Search for {%- for message in messages -%} loops + for loop_ast in root.find_all(jinja2.nodes.For): + loop_iter = loop_ast.iter + loop_target = loop_ast.target + + for varname in messages_varnames: + if _is_var_or_elems_access(loop_iter, varname): + assert isinstance(loop_target, jinja2.nodes.Name) + yield loop_ast, loop_target.name + break + + +def _iter_nodes_assign_content_item(root: jinja2.nodes.Node): + message_varnames = [ + varname for _, varname in _iter_nodes_assign_messages_item(root) + ] + + # Search for {%- for content in message['content'] -%} loops + for loop_ast in root.find_all(jinja2.nodes.For): + loop_iter = loop_ast.iter + loop_target = loop_ast.target + + for varname in message_varnames: + if _is_var_or_elems_access(loop_iter, varname, "content"): + assert isinstance(loop_target, jinja2.nodes.Name) + yield loop_ast, loop_target.name + break + + +def _try_extract_ast(chat_template: str) -> jinja2.nodes.Template | None: + import transformers.utils.chat_template_utils as hf_chat_utils + + try: + jinja_compiled = hf_chat_utils._compile_jinja_template(chat_template) + return jinja_compiled.environment.parse(chat_template) + except Exception: + logger.exception("Error when compiling Jinja template") + return None + + +@lru_cache(maxsize=32) +def _detect_content_format( + chat_template: str, + *, + default: ChatTemplateContentFormat, +) -> ChatTemplateContentFormat: + jinja_ast = _try_extract_ast(chat_template) + if jinja_ast is None: + return default + + try: + next(_iter_nodes_assign_content_item(jinja_ast)) + except StopIteration: + return "string" + except Exception: + logger.exception("Error when parsing AST of Jinja template") + return default + else: + return "openai" + + +def _resolve_chat_template_content_format( + chat_template: str | None, + tools: list[dict[str, Any]] | None, + tokenizer: HfTokenizer, + *, + model_config: "ModelConfig", +) -> ChatTemplateContentFormat: + resolved_chat_template = resolve_chat_template( + tokenizer, + chat_template=chat_template, + tools=tools, + model_config=model_config, + ) + + jinja_text = ( + resolved_chat_template + if isinstance(resolved_chat_template, str) + else load_chat_template(chat_template, is_literal=True) + ) + + detected_format = ( + "string" + if jinja_text is None + else _detect_content_format(jinja_text, default="string") + ) + + return detected_format + + +@lru_cache +def _log_chat_template_content_format( + chat_template: str | None, # For caching purposes + given_format: ChatTemplateContentFormatOption, + detected_format: ChatTemplateContentFormatOption, +): + logger.info( + "Detected the chat template content format to be '%s'. " + "You can set `--chat-template-content-format` to override this.", + detected_format, + ) + + if given_format != "auto" and given_format != detected_format: + logger.warning( + "You specified `--chat-template-content-format %s` " + "which is different from the detected format '%s'. " + "If our automatic detection is incorrect, please consider " + "opening a GitHub issue so that we can improve it: " + "https://github.com/vllm-project/vllm/issues/new/choose", + given_format, + detected_format, + ) + + +def resolve_chat_template_content_format( + chat_template: str | None, + tools: list[dict[str, Any]] | None, + given_format: ChatTemplateContentFormatOption, + tokenizer: HfTokenizer, + *, + model_config: "ModelConfig", +) -> ChatTemplateContentFormat: + if given_format != "auto": + return given_format + + detected_format = _resolve_chat_template_content_format( + chat_template, + tools, + tokenizer, + model_config=model_config, + ) + + _log_chat_template_content_format( + chat_template, + given_format=given_format, + detected_format=detected_format, + ) + + return detected_format + + +# adapted from https://github.com/huggingface/transformers/blob/v4.56.2/src/transformers/utils/chat_template_utils.py#L398-L412 +# only preserve the parse function used to resolve chat template kwargs +class AssistantTracker(jinja2.ext.Extension): + tags = {"generation"} + + def parse(self, parser: jinja2.parser.Parser) -> jinja2.nodes.Node: + lineno = next(parser.stream).lineno + body = parser.parse_statements(("name:endgeneration",), drop_needle=True) + call = self.call_method("_generation_support") + call_block = jinja2.nodes.CallBlock(call, [], [], body) + return call_block.set_lineno(lineno) + + +def _resolve_chat_template_kwargs(chat_template: str) -> Set[str]: + env = jinja2.sandbox.ImmutableSandboxedEnvironment( + trim_blocks=True, + lstrip_blocks=True, + extensions=[AssistantTracker, jinja2.ext.loopcontrols], + ) + parsed_content = env.parse(chat_template) + template_vars = jinja2.meta.find_undeclared_variables(parsed_content) + return template_vars + + +_cached_resolve_chat_template_kwargs = lru_cache(_resolve_chat_template_kwargs) + + +@lru_cache +def _get_hf_base_chat_template_params() -> frozenset[str]: + from transformers import PreTrainedTokenizer + + # Get standard parameters from HuggingFace's base tokenizer class. + # This dynamically extracts parameters from PreTrainedTokenizer's + # apply_chat_template method, ensuring compatibility with tokenizers + # that use **kwargs to receive standard parameters. + + # Read signature from HF's base class - the single source of truth + base_sig = inspect.signature(PreTrainedTokenizer.apply_chat_template) + + # Exclude VAR_KEYWORD (**kwargs) and VAR_POSITIONAL (*args) placeholders + return frozenset( + p.name + for p in base_sig.parameters.values() + if p.kind + not in (inspect.Parameter.VAR_KEYWORD, inspect.Parameter.VAR_POSITIONAL) + ) + + +def resolve_chat_template_kwargs( + tokenizer: HfTokenizer, + chat_template: str, + chat_template_kwargs: dict[str, Any], + raise_on_unexpected: bool = True, +) -> dict[str, Any]: + # We exclude chat_template from kwargs here, because + # chat template has been already resolved at this stage + unexpected_vars = {"chat_template", "tokenize"} + if raise_on_unexpected and ( + unexpected_in_kwargs := unexpected_vars & chat_template_kwargs.keys() + ): + raise ValueError( + "Found unexpected chat template kwargs from request: " + f"{unexpected_in_kwargs}" + ) + + fn_kw = { + k + for k in chat_template_kwargs + if supports_kw(tokenizer.apply_chat_template, k, allow_var_kwargs=False) + } + template_vars = _cached_resolve_chat_template_kwargs(chat_template) + + # Allow standard HF parameters even if tokenizer uses **kwargs to receive them + hf_base_params = _get_hf_base_chat_template_params() + + accept_vars = (fn_kw | template_vars | hf_base_params) - unexpected_vars + return {k: v for k, v in chat_template_kwargs.items() if k in accept_vars} + + +def safe_apply_chat_template( + model_config: "ModelConfig", + tokenizer: HfTokenizer, + conversation: list[ConversationMessage], + *, + tools: list[dict[str, Any]] | None = None, + chat_template: str | None = None, + tokenize: bool = True, + **kwargs, +) -> str | list[int]: + chat_template = resolve_chat_template( + tokenizer, + chat_template=chat_template, + tools=tools, + model_config=model_config, + ) + if chat_template is None: + raise ValueError( + "As of transformers v4.44, default chat template is no longer " + "allowed, so you must provide a chat template if the tokenizer " + "does not define one." + ) + + resolved_kwargs = resolve_chat_template_kwargs( + tokenizer=tokenizer, + chat_template=chat_template, + chat_template_kwargs=kwargs, + ) + + try: + return tokenizer.apply_chat_template( + conversation=conversation, # type: ignore[arg-type] + tools=tools, # type: ignore[arg-type] + chat_template=chat_template, + tokenize=tokenize, + **resolved_kwargs, + ) + # External library exceptions can sometimes occur despite the framework's + # internal exception management capabilities. + except Exception as e: + # Log and report any library-related exceptions for further + # investigation. + logger.exception( + "An error occurred in `transformers` while applying chat template" + ) + raise ValueError(str(e)) from e + + +class HfRenderer(RendererLike): + @classmethod + def from_config( + cls, + config: ModelConfig, + tokenizer_kwargs: dict[str, Any], + ) -> "RendererLike": + return cls(config, tokenizer_kwargs) + + def __init__( + self, + config: ModelConfig, + tokenizer_kwargs: dict[str, Any], + ) -> None: + super().__init__() + + self.config = config + + if config.skip_tokenizer_init: + tokenizer = None + else: + tokenizer = get_tokenizer(HfTokenizer, **tokenizer_kwargs) + + self._tokenizer = tokenizer + + @property + def tokenizer(self) -> HfTokenizer | None: + return self._tokenizer + + # NOTE: Remove this once LLM.tokenizer.setter is removed + @tokenizer.setter + def tokenizer(self, tokenizer: HfTokenizer | None) -> None: + self._tokenizer = tokenizer + + def get_tokenizer(self) -> HfTokenizer: + tokenizer = self.tokenizer + if tokenizer is None: + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") + + return tokenizer + + def render_messages( + self, + messages: list[ChatCompletionMessageParam], + chat_template_content_format: ChatTemplateContentFormat, + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + model_config = self.config + tokenizer = self.get_tokenizer() + + conversation, mm_data, mm_uuids = parse_chat_messages( + messages, + model_config, + content_format=resolve_chat_template_content_format( + chat_template=kwargs.get("chat_template"), + tools=kwargs.get("tools"), + given_format=chat_template_content_format, + tokenizer=tokenizer, + model_config=model_config, + ), + ) + + prompt_raw = safe_apply_chat_template( + model_config, + tokenizer, + conversation, + **kwargs, + ) + if isinstance(prompt_raw, str): + prompt = TextPrompt(prompt=prompt_raw) + else: + prompt = TokensPrompt(prompt_token_ids=prompt_raw) + + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt + + async def render_messages_async( + self, + messages: list[ChatCompletionMessageParam], + chat_template_content_format: ChatTemplateContentFormat, + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + model_config = self.config + tokenizer = self.get_tokenizer() + + conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + messages, + model_config, + content_format=resolve_chat_template_content_format( + chat_template=kwargs.get("chat_template"), + tools=kwargs.get("tools"), + given_format=chat_template_content_format, + tokenizer=tokenizer, + model_config=model_config, + ), + ) + + prompt_raw = safe_apply_chat_template( + model_config, + tokenizer, + conversation, + **kwargs, + ) + if isinstance(prompt_raw, str): + prompt = TextPrompt(prompt=prompt_raw) + else: + prompt = TokensPrompt(prompt_token_ids=prompt_raw) + + if mm_data_future is not None: + prompt["multi_modal_data"] = await mm_data_future + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py new file mode 100644 index 000000000000..5268608674c2 --- /dev/null +++ b/vllm/renderers/mistral.py @@ -0,0 +1,147 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from concurrent.futures import ThreadPoolExecutor +from typing import Any + +from vllm.config import ModelConfig +from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, + ConversationMessage, + parse_chat_messages, + parse_chat_messages_futures, +) +from vllm.inputs import TextPrompt, TokensPrompt +from vllm.logger import init_logger +from vllm.tokenizers import get_tokenizer +from vllm.tokenizers.mistral import MistralTokenizer +from vllm.utils.async_utils import make_async + +from .protocol import RendererLike + +logger = init_logger(__name__) + + +def safe_apply_chat_template( + tokenizer: MistralTokenizer, + messages: list[ChatCompletionMessageParam], + **kwargs, +) -> str | list[int]: + from mistral_common.exceptions import MistralCommonException + + try: + return tokenizer.apply_chat_template(messages, **kwargs) + # mistral-common uses assert statements to stop processing of input + # if input does not comply with the expected format. + # We convert those assertion errors to ValueErrors so they can be + # properly caught in the preprocessing_input step + except (AssertionError, MistralCommonException) as e: + raise ValueError(str(e)) from e + + # External library exceptions can sometimes occur despite the framework's + # internal exception management capabilities. + except Exception as e: + # Log and report any library-related exceptions for further + # investigation. + logger.exception( + "An error occurred in `mistral_common` while applying chat template" + ) + raise ValueError(str(e)) from e + + +class MistralRenderer(RendererLike): + @classmethod + def from_config( + cls, + config: ModelConfig, + tokenizer_kwargs: dict[str, Any], + ) -> "RendererLike": + return cls(config, tokenizer_kwargs) + + def __init__( + self, + config: ModelConfig, + tokenizer_kwargs: dict[str, Any], + ) -> None: + super().__init__() + + self.config = config + + if config.skip_tokenizer_init: + tokenizer = None + else: + tokenizer = get_tokenizer(MistralTokenizer, **tokenizer_kwargs) + + self._tokenizer = tokenizer + + self._apply_chat_template_executor = ThreadPoolExecutor(max_workers=1) + self._apply_chat_template_async = make_async( + safe_apply_chat_template, executor=self._apply_chat_template_executor + ) + + @property + def tokenizer(self) -> MistralTokenizer | None: + return self._tokenizer + + # NOTE: Remove this once LLM.tokenizer.setter is removed + @tokenizer.setter + def tokenizer(self, tokenizer: MistralTokenizer | None) -> None: + self._tokenizer = tokenizer + + def get_tokenizer(self) -> MistralTokenizer: + tokenizer = self.tokenizer + if tokenizer is None: + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") + + return tokenizer + + def render_messages( + self, + messages: list[ChatCompletionMessageParam], + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + tokenizer = self.get_tokenizer() + conversation, mm_data, mm_uuids = parse_chat_messages( + messages, + self.config, + content_format="string", + ) + + prompt_raw = safe_apply_chat_template(tokenizer, messages, **kwargs) + if isinstance(prompt_raw, str): + prompt = TextPrompt(prompt=prompt_raw) + else: + prompt = TokensPrompt(prompt_token_ids=prompt_raw) + + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt + + async def render_messages_async( + self, + messages: list[ChatCompletionMessageParam], + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + tokenizer = self.get_tokenizer() + conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + messages, + self.config, + content_format="string", + ) + + prompt_raw = await self._apply_chat_template_async( + tokenizer, messages, **kwargs + ) + if isinstance(prompt_raw, str): + prompt = TextPrompt(prompt=prompt_raw) + else: + prompt = TokensPrompt(prompt_token_ids=prompt_raw) + + if mm_data_future is not None: + prompt["multi_modal_data"] = await mm_data_future + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt diff --git a/vllm/renderers/protocol.py b/vllm/renderers/protocol.py new file mode 100644 index 000000000000..641121f880bc --- /dev/null +++ b/vllm/renderers/protocol.py @@ -0,0 +1,53 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import TYPE_CHECKING, Any, Protocol + +from vllm.inputs import TextPrompt, TokensPrompt +from vllm.tokenizers import TokenizerLike + +if TYPE_CHECKING: + from vllm.config import ModelConfig + from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, + ConversationMessage, + ) + + +class RendererLike(Protocol): + @classmethod + def from_config( + cls, + config: "ModelConfig", + tokenizer_kwargs: dict[str, Any], + ) -> "RendererLike": + raise NotImplementedError + + @property + def tokenizer(self) -> TokenizerLike | None: + raise NotImplementedError + + # NOTE: Remove this once LLM.tokenizer.setter is removed + @tokenizer.setter + def tokenizer(self, tokenizer: TokenizerLike | None) -> None: + self._tokenizer = tokenizer + + def get_tokenizer(self) -> TokenizerLike: + tokenizer = self.tokenizer + if tokenizer is None: + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") + + return tokenizer + + def render_messages( + self, + messages: list["ChatCompletionMessageParam"], + **kwargs, + ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt]: + raise NotImplementedError + + async def render_messages_async( + self, + messages: list["ChatCompletionMessageParam"], + **kwargs, + ) -> tuple[list["ConversationMessage"], TextPrompt | TokensPrompt]: + return self.render_messages(messages, **kwargs) diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py new file mode 100644 index 000000000000..376d9e3ab6f8 --- /dev/null +++ b/vllm/renderers/registry.py @@ -0,0 +1,110 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib.util +from typing import TYPE_CHECKING, Any, TypeVar + +from typing_extensions import assert_never + +from vllm.logger import init_logger +from vllm.transformers_utils.repo_utils import list_filtered_repo_files +from vllm.utils.import_utils import resolve_obj_by_qualname + +from .protocol import RendererLike + +if TYPE_CHECKING: + from vllm.config import ModelConfig + +logger = init_logger(__name__) + +_T = TypeVar("_T", bound=type[RendererLike]) + + +class RendererRegistry: + # Renderer name -> (renderer module, renderer class) + REGISTRY: dict[str, tuple[str, str]] = { + "deepseekv32": ("vllm.renderers.deepseekv32", "DeepseekV32Renderer"), + "hf": ("vllm.renderers.hf", "HfRenderer"), + "mistral": ("vllm.renderers.mistral", "MistralRenderer"), + "terratorch": ("vllm.renderers.terratorch", "TerratorchRenderer"), + } + + @staticmethod + def register(renderer_mode: str, module: str, class_name: str) -> None: + if renderer_mode in RendererRegistry.REGISTRY: + logger.warning( + "%s.%s is already registered for renderer_mode=%r. " + "It is overwritten by the new one.", + module, + class_name, + renderer_mode, + ) + + RendererRegistry.REGISTRY[renderer_mode] = (module, class_name) + + return None + + @staticmethod + def init_renderer( + renderer_mode: str, + config: "ModelConfig", + tokenizer_kwargs: dict[str, Any], + ) -> RendererLike: + if renderer_mode not in RendererRegistry.REGISTRY: + raise ValueError(f"No renderer registered for {renderer_mode=!r}.") + + module, class_name = RendererRegistry.REGISTRY[renderer_mode] + logger.debug_once(f"Loading {class_name} for {renderer_mode=!r}") + + cls_: type[RendererLike] = resolve_obj_by_qualname(f"{module}.{class_name}") + return cls_.from_config(config, tokenizer_kwargs) + + +def renderer_from_config(config: "ModelConfig"): + tokenizer_name = config.tokenizer + tokenizer_mode = config.tokenizer_mode + tokenizer_revision = config.tokenizer_revision + trust_remote_code = config.trust_remote_code + tokenizer_kwargs = dict[str, Any]() + + runner_type = config.runner_type + if runner_type == "generate" or runner_type == "draft": + tokenizer_kwargs["truncation_side"] = "left" + elif runner_type == "pooling": + tokenizer_kwargs["truncation_side"] = "right" + else: + assert_never(runner_type) + + tokenizer_mode = config.tokenizer_mode + if tokenizer_mode == "slow": + if tokenizer_kwargs.get("use_fast", False): + raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") + + tokenizer_mode = "hf" + tokenizer_kwargs["use_fast"] = False + + # Try to use official Mistral tokenizer if possible + if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"): + allow_patterns = ["tekken.json", "tokenizer.model.v*"] + files_list = list_filtered_repo_files( + model_name_or_path=str(tokenizer_name), + allow_patterns=allow_patterns, + revision=tokenizer_revision, + ) + if len(files_list) > 0: + tokenizer_mode = "mistral" + + # Fallback to HF tokenizer + if tokenizer_mode == "auto": + tokenizer_mode = "hf" + + tokenizer_kwargs = dict[str, Any]( + trust_remote_code=trust_remote_code, + revision=tokenizer_revision, + **tokenizer_kwargs, + ) + + return RendererRegistry.init_renderer( + tokenizer_mode, + config, + tokenizer_kwargs, + ) diff --git a/vllm/renderers/terratorch.py b/vllm/renderers/terratorch.py new file mode 100644 index 000000000000..d1d7c842214c --- /dev/null +++ b/vllm/renderers/terratorch.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm.config import ModelConfig +from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, + ConversationMessage, + parse_chat_messages, + parse_chat_messages_futures, +) +from vllm.inputs import TextPrompt, TokensPrompt +from vllm.logger import init_logger +from vllm.tokenizers import TokenizerLike + +from .protocol import RendererLike + +logger = init_logger(__name__) + + +class TerratorchRenderer(RendererLike): + @classmethod + def from_config(cls, config: ModelConfig) -> "RendererLike": + return cls(config) + + def __init__(self, config: ModelConfig) -> None: + super().__init__() + + self.config = config + + if not config.skip_tokenizer_init: + raise ValueError("Terratorch renderer requires `skip_tokenizer_init=True`") + + @property + def tokenizer(self) -> TokenizerLike | None: + return None + + def get_tokenizer(self) -> TokenizerLike: + raise ValueError("Tokenizer not available for Terratorch renderer") + + def render_messages( + self, + messages: list[ChatCompletionMessageParam], + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + model_config = self.config + + conversation, mm_data, mm_uuids = parse_chat_messages( + messages, + model_config, + content_format="string", + ) + + prompt = TokensPrompt(prompt_token_ids=[1]) + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt + + async def render_messages_async( + self, + messages: list[ChatCompletionMessageParam], + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + model_config = self.config + + conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + messages, + model_config, + content_format="string", + ) + + prompt = TokensPrompt(prompt_token_ids=[1]) # Dummy token IDs + if mm_data_future is not None: + prompt["multi_modal_data"] = await mm_data_future + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt diff --git a/vllm/tokenizers/__init__.py b/vllm/tokenizers/__init__.py index 67a6d7c8eb3d..31e74b1a16e2 100644 --- a/vllm/tokenizers/__init__.py +++ b/vllm/tokenizers/__init__.py @@ -1,9 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from .deepseekv32 import DeepseekV32Tokenizer -from .hf import HfTokenizer -from .mistral import MistralTokenizer from .protocol import TokenizerLike from .registry import ( TokenizerRegistry, @@ -15,12 +12,9 @@ __all__ = [ "TokenizerLike", - "HfTokenizer", - "MistralTokenizer", "TokenizerRegistry", "cached_get_tokenizer", "get_tokenizer", "cached_tokenizer_from_config", "init_tokenizer_from_config", - "DeepseekV32Tokenizer", ] diff --git a/vllm/tokenizers/deepseekv32.py b/vllm/tokenizers/deepseekv32.py index b0490dacbe2d..43dbb2b5c56c 100644 --- a/vllm/tokenizers/deepseekv32.py +++ b/vllm/tokenizers/deepseekv32.py @@ -2,22 +2,18 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from pathlib import Path +from typing import Any from transformers import BatchEncoding +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam + from .deepseek_v32_encoding import encode_messages -from .hf import HfTokenizer, TokenizerLike -from .registry import TokenizerRegistry +from .hf import CachedHfTokenizer +from .protocol import TokenizerLike -@TokenizerRegistry.register("deepseek_v32") -class DeepseekV32Tokenizer(HfTokenizer): - def __init__(self, tokenizer: TokenizerLike): - self.tokenizer = tokenizer - self.name_or_path = ( - tokenizer.name_or_path if hasattr(tokenizer, "name_or_path") else "" - ) - +class DeepseekV32Tokenizer(CachedHfTokenizer): @classmethod def from_pretrained( cls, @@ -38,7 +34,18 @@ def from_pretrained( ) return DeepseekV32Tokenizer(tokenizer) - def apply_chat_template(self, messages, tools=None, **kwargs): + def __init__(self, tokenizer: TokenizerLike) -> None: + super().__init__() + + self.tokenizer = tokenizer + self.name_or_path = getattr(tokenizer, "name_or_path", "") + + def apply_chat_template( + self, + messages: list["ChatCompletionMessageParam"], + tools: list[dict[str, Any]] | None = None, + **kwargs, + ) -> str: thinking = kwargs.get("thinking", False) thinking_mode = "thinking" if not thinking: @@ -48,7 +55,7 @@ def apply_chat_template(self, messages, tools=None, **kwargs): drop_thinking = True if tools is not None and len(tools) > 0: messages.insert(0, {"role": "system"}) - messages[0]["tools"] = tools + messages[0]["tools"] = tools # type: ignore[typeddict-unknown-key] drop_thinking = False encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking) prompt_str = encode_messages(messages, **encode_config) # type: ignore diff --git a/vllm/tokenizers/hf.py b/vllm/tokenizers/hf.py index 344507312038..c455d80a3fbc 100644 --- a/vllm/tokenizers/hf.py +++ b/vllm/tokenizers/hf.py @@ -3,22 +3,18 @@ import contextlib import copy from pathlib import Path -from typing import TYPE_CHECKING +from typing import TypeAlias -from transformers import AutoTokenizer +from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config from .protocol import TokenizerLike -from .registry import TokenizerRegistry -if TYPE_CHECKING: - from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast +HfTokenizer: TypeAlias = PreTrainedTokenizer | PreTrainedTokenizerFast -def get_cached_tokenizer( - tokenizer: "PreTrainedTokenizer | PreTrainedTokenizerFast", -) -> TokenizerLike: +def get_cached_tokenizer(tokenizer: HfTokenizer) -> HfTokenizer: """ By default, transformers will recompute multiple tokenizer properties each time they are called, leading to a significant slowdown. @@ -65,11 +61,10 @@ def __reduce__(self): CachedTokenizer.__name__ = f"Cached{tokenizer.__class__.__name__}" cached_tokenizer.__class__ = CachedTokenizer - return cached_tokenizer # type: ignore + return cached_tokenizer -@TokenizerRegistry.register("hf") -class HfTokenizer(TokenizerLike): +class CachedHfTokenizer(TokenizerLike): @classmethod def from_pretrained( cls, @@ -79,7 +74,9 @@ def from_pretrained( revision: str | None = None, download_dir: str | None = None, **kwargs, - ) -> "TokenizerLike": + ) -> HfTokenizer: + from transformers import AutoTokenizer + try: tokenizer = AutoTokenizer.from_pretrained( path_or_repo_id, diff --git a/vllm/tokenizers/mistral.py b/vllm/tokenizers/mistral.py index 1f44037dd55e..534b0da484a5 100644 --- a/vllm/tokenizers/mistral.py +++ b/vllm/tokenizers/mistral.py @@ -3,10 +3,11 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, cast +from vllm.entrypoints.chat_utils import ChatCompletionMessageParam +from vllm.entrypoints.openai.protocol import ChatCompletionRequest from vllm.logger import init_logger from .protocol import TokenizerLike -from .registry import TokenizerRegistry if TYPE_CHECKING: from mistral_common.protocol.instruct.request import ( @@ -15,9 +16,6 @@ from mistral_common.tokens.tokenizers.tekken import Tekkenizer from transformers import BatchEncoding - from vllm.entrypoints.chat_utils import ChatCompletionMessageParam - from vllm.entrypoints.openai.protocol import ChatCompletionRequest - try: # Transformers v5 from transformers.tokenization_mistral_common import MistralCommonBackend @@ -201,7 +199,6 @@ def _tekken_token_to_id(tokenizer: "Tekkenizer", t: str | bytes) -> int: return tokenizer.unk_id -@TokenizerRegistry.register("mistral") class MistralTokenizer(TokenizerLike): @classmethod def from_pretrained( diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 1d44feeee500..35ac00f69f5c 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -1,10 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import importlib.util -from collections.abc import Callable from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, TypeVar, overload +from typing import TYPE_CHECKING, Any, TypeVar import huggingface_hub from typing_extensions import assert_never @@ -18,7 +16,6 @@ is_remote_gguf, split_remote_gguf, ) -from vllm.transformers_utils.repo_utils import list_filtered_repo_files from vllm.utils.import_utils import resolve_obj_by_qualname from .protocol import TokenizerLike @@ -28,41 +25,19 @@ logger = init_logger(__name__) -_T = TypeVar("_T", bound=type[TokenizerLike]) +_T = TypeVar("_T", bound=TokenizerLike) class TokenizerRegistry: - # Tokenizer name -> tokenizer_cls or (tokenizer module, tokenizer class) - REGISTRY: dict[str, type[TokenizerLike] | tuple[str, str]] = {} + # Tokenizer name -> (tokenizer module, tokenizer class) + REGISTRY: dict[str, tuple[str, str]] = { + "deepseekv32": ("vllm.tokenizers.deepseekv32", "DeepseekV32Tokenizer"), + "hf": ("vllm.tokenizers.hf", "CachedHfTokenizer"), + "mistral": ("vllm.tokenizers.mistral", "MistralTokenizer"), + } - # In-tree tokenizers @staticmethod - @overload - def register(tokenizer_mode: str) -> Callable[[_T], _T]: ... - - # OOT tokenizers - @staticmethod - @overload - def register(tokenizer_mode: str, module: str, class_name: str) -> None: ... - - @staticmethod - def register( - tokenizer_mode: str, - module: str | None = None, - class_name: str | None = None, - ) -> Callable[[_T], _T] | None: - # In-tree tokenizers - if module is None or class_name is None: - - def wrapper(tokenizer_cls: _T) -> _T: - assert tokenizer_mode not in TokenizerRegistry.REGISTRY - TokenizerRegistry.REGISTRY[tokenizer_mode] = tokenizer_cls - - return tokenizer_cls - - return wrapper - - # OOT tokenizers + def register(tokenizer_mode: str, module: str, class_name: str) -> None: if tokenizer_mode in TokenizerRegistry.REGISTRY: logger.warning( "%s.%s is already registered for tokenizer_mode=%r. " @@ -77,30 +52,26 @@ def wrapper(tokenizer_cls: _T) -> _T: return None @staticmethod - def get_tokenizer(tokenizer_mode: str, *args, **kwargs) -> "TokenizerLike": + def init_tokenizer(tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: if tokenizer_mode not in TokenizerRegistry.REGISTRY: raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.") - item = TokenizerRegistry.REGISTRY[tokenizer_mode] - if isinstance(item, type): - return item.from_pretrained(*args, **kwargs) - - module, class_name = item + module, class_name = TokenizerRegistry.REGISTRY[tokenizer_mode] logger.debug_once(f"Loading {class_name} for {tokenizer_mode=!r}") - class_ = resolve_obj_by_qualname(f"{module}.{class_name}") - return class_.from_pretrained(*args, **kwargs) + cls_: type[TokenizerLike] = resolve_obj_by_qualname(f"{module}.{class_name}") + return cls_.from_pretrained(*args, **kwargs) def get_tokenizer( + tokenizer_cls: type[_T], tokenizer_name: str | Path, *args, - tokenizer_mode: str = "auto", trust_remote_code: bool = False, revision: str | None = None, download_dir: str | None = None, **kwargs, -) -> TokenizerLike: +) -> _T: """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" if envs.VLLM_USE_MODELSCOPE: # download model from ModelScope hub, @@ -125,16 +96,6 @@ def get_tokenizer( ) tokenizer_name = tokenizer_path - if tokenizer_mode == "slow": - if kwargs.get("use_fast", False): - raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") - - tokenizer_mode = "hf" - kwargs["use_fast"] = False - - if "truncation_side" not in kwargs: - kwargs["truncation_side"] = "left" - # Separate model folder from file path for GGUF models if is_gguf(tokenizer_name): if check_gguf_file(tokenizer_name): @@ -150,56 +111,22 @@ def get_tokenizer( ) kwargs["gguf_file"] = gguf_file - # Try to use official Mistral tokenizer if possible - if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"): - allow_patterns = ["tekken.json", "tokenizer.model.v*"] - files_list = list_filtered_repo_files( - model_name_or_path=str(tokenizer_name), - allow_patterns=allow_patterns, - revision=revision, - ) - if len(files_list) > 0: - tokenizer_mode = "mistral" - - # Fallback to HF tokenizer - if tokenizer_mode == "auto": - tokenizer_mode = "hf" - tokenizer_args = (tokenizer_name, *args) - tokenizer_kwargs = dict( + tokenizer_kwargs = dict[str, Any]( trust_remote_code=trust_remote_code, revision=revision, download_dir=download_dir, **kwargs, ) - if tokenizer_mode == "custom": - logger.warning_once( - "TokenizerRegistry now uses `tokenizer_mode` as the registry key " - "instead of `tokenizer_name`. " - "Please update the definition of `.from_pretrained` in " - "your custom tokenizer to accept `args=%s`, `kwargs=%s`. " - "Then, you can pass `tokenizer_mode=%r` instead of " - "`tokenizer_mode='custom'` when initializing vLLM.", - tokenizer_args, - str(tokenizer_kwargs), - tokenizer_name, - ) - - tokenizer_mode = str(tokenizer_name) - - tokenizer = TokenizerRegistry.get_tokenizer( - tokenizer_mode, - *tokenizer_args, - **tokenizer_kwargs, - ) + tokenizer = tokenizer_cls.from_pretrained(*tokenizer_args, **tokenizer_kwargs) if not tokenizer.is_fast: logger.warning( "Using a slow tokenizer. This might cause a significant " "slowdown. Consider using a fast tokenizer instead." ) - return tokenizer + return tokenizer # type: ignore cached_get_tokenizer = lru_cache(get_tokenizer) @@ -216,6 +143,9 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs): def init_tokenizer_from_config(model_config: "ModelConfig"): + if model_config.skip_tokenizer_init: + return None + runner_type = model_config.runner_type if runner_type == "generate" or runner_type == "draft": truncation_side = "left" @@ -224,10 +154,4 @@ def init_tokenizer_from_config(model_config: "ModelConfig"): else: assert_never(runner_type) - return get_tokenizer( - model_config.tokenizer, - tokenizer_mode=model_config.tokenizer_mode, - trust_remote_code=model_config.trust_remote_code, - revision=model_config.tokenizer_revision, - truncation_side=truncation_side, - ) + return cached_tokenizer_from_config(model_config, truncation_side=truncation_side) diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index fd7e04dc0208..cd7ae5b6072d 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -24,9 +24,10 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput from vllm.plugins.io_processors import get_io_processor from vllm.pooling_params import PoolingParams +from vllm.renderers import RendererLike from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask -from vllm.tokenizers import TokenizerLike, init_tokenizer_from_config +from vllm.tokenizers import TokenizerLike from vllm.tracing import init_tracer from vllm.transformers_utils.config import maybe_register_config_serialize_by_value from vllm.usage.usage_lib import UsageContext @@ -108,12 +109,7 @@ def __init__( "enabling logging without default stat loggers." ) - if self.model_config.skip_tokenizer_init: - tokenizer = None - else: - tokenizer = init_tokenizer_from_config(self.model_config) - - self.input_processor = InputProcessor(self.vllm_config, tokenizer) + self.input_processor = InputProcessor(self.vllm_config) self.io_processor = get_io_processor( self.vllm_config, self.model_config.io_processor_plugin, @@ -714,12 +710,11 @@ def tokenizer(self, tokenizer: TokenizerLike | None) -> None: self.input_processor.tokenizer = tokenizer async def get_tokenizer(self) -> TokenizerLike: - if self.tokenizer is None: - raise ValueError( - "Unable to get tokenizer because `skip_tokenizer_init=True`" - ) + return self.input_processor.get_tokenizer() - return self.tokenizer + @property + def renderer(self) -> RendererLike: + return self.input_processor.renderer async def is_tracing_enabled(self) -> bool: return self.observability_config.otlp_traces_endpoint is not None # type: ignore diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index e6a94f4e3de5..6deca1a6dad8 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -18,8 +18,10 @@ from vllm.multimodal.processing import EncDecMultiModalProcessor from vllm.multimodal.utils import argsort_mm_positions from vllm.pooling_params import PoolingParams +from vllm.renderers import RendererLike from vllm.sampling_params import SamplingParams -from vllm.tokenizers import MistralTokenizer, TokenizerLike +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils import length_from_prompt_token_ids_or_embeds from vllm.v1.engine import EngineCoreRequest from vllm.v1.metrics.stats import MultiModalCacheStats @@ -39,7 +41,6 @@ class InputProcessor: def __init__( self, vllm_config: VllmConfig, - tokenizer: TokenizerLike | None, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, ) -> None: self.vllm_config = vllm_config @@ -55,7 +56,6 @@ def __init__( self.input_preprocessor = InputPreprocessor( self.model_config, - tokenizer, mm_registry, mm_processor_cache=self.mm_processor_cache, ) @@ -68,6 +68,13 @@ def tokenizer(self) -> TokenizerLike | None: def tokenizer(self, tokenizer: TokenizerLike | None) -> None: self.input_preprocessor.tokenizer = tokenizer + def get_tokenizer(self) -> TokenizerLike: + return self.input_preprocessor.get_tokenizer() + + @property + def renderer(self) -> RendererLike: + return self.input_preprocessor.renderer + def _validate_logprobs( self, params: SamplingParams, diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index 4c3129100547..c8f8fe33005d 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -21,9 +21,10 @@ from vllm.outputs import PoolingRequestOutput, RequestOutput from vllm.plugins.io_processors import get_io_processor from vllm.pooling_params import PoolingParams +from vllm.renderers import RendererLike from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask -from vllm.tokenizers import TokenizerLike, init_tokenizer_from_config +from vllm.tokenizers import TokenizerLike from vllm.tracing import init_tracer from vllm.usage.usage_lib import UsageContext from vllm.v1.engine import EngineCoreRequest @@ -83,12 +84,7 @@ def __init__( self.dp_group = None self.should_execute_dummy_batch = False - if self.model_config.skip_tokenizer_init: - tokenizer = None - else: - tokenizer = init_tokenizer_from_config(self.model_config) - - self.input_processor = InputProcessor(self.vllm_config, tokenizer) + self.input_processor = InputProcessor(self.vllm_config) self.io_processor = get_io_processor( self.vllm_config, self.model_config.io_processor_plugin, @@ -363,12 +359,11 @@ def tokenizer(self, tokenizer: TokenizerLike | None) -> None: self.input_processor.tokenizer = tokenizer def get_tokenizer(self) -> TokenizerLike: - if self.tokenizer is None: - raise ValueError( - "Unable to get tokenizer because `skip_tokenizer_init=True`" - ) + return self.input_processor.get_tokenizer() - return self.tokenizer + @property + def renderer(self) -> RendererLike: + return self.input_processor.renderer def do_log_stats(self) -> None: """Log stats if logging is enabled.""" diff --git a/vllm/v1/structured_output/backend_xgrammar.py b/vllm/v1/structured_output/backend_xgrammar.py index f8a2df43dd90..fa852b2fb79f 100644 --- a/vllm/v1/structured_output/backend_xgrammar.py +++ b/vllm/v1/structured_output/backend_xgrammar.py @@ -10,7 +10,7 @@ import vllm.envs from vllm.logger import init_logger from vllm.sampling_params import SamplingParams -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.import_utils import LazyLoader from vllm.v1.structured_output.backend_types import ( StructuredOutputBackend, From b7222cb570ea2f5cf992624140d3cc2b817fad93 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 08:39:40 +0000 Subject: [PATCH 02/65] Simplify Signed-off-by: DarkLight1337 --- vllm/engine/protocol.py | 6 --- vllm/entrypoints/openai/api_server.py | 6 +-- vllm/entrypoints/openai/serving_completion.py | 5 +-- vllm/entrypoints/pooling/embed/serving.py | 12 +++--- vllm/entrypoints/pooling/pooling/serving.py | 5 +-- vllm/entrypoints/pooling/score/serving.py | 3 +- vllm/entrypoints/serve/tokenize/serving.py | 7 ++- vllm/entrypoints/utils.py | 43 ------------------- vllm/v1/engine/async_llm.py | 2 +- 9 files changed, 15 insertions(+), 74 deletions(-) diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index e786db52e2a1..461c1f193c87 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -14,7 +14,6 @@ from vllm.renderers import RendererLike from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask -from vllm.tokenizers import TokenizerLike from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.input_processor import InputProcessor @@ -89,11 +88,6 @@ async def abort(self, request_id: str | Iterable[str]) -> None: """ ... - @abstractmethod - async def get_tokenizer(self) -> TokenizerLike: - """Get the tokenizer""" - ... - @abstractmethod async def is_tracing_enabled(self) -> bool: ... diff --git a/vllm/entrypoints/openai/api_server.py b/vllm/entrypoints/openai/api_server.py index 7be601d824f3..432b350b8ee8 100644 --- a/vllm/entrypoints/openai/api_server.py +++ b/vllm/entrypoints/openai/api_server.py @@ -41,6 +41,7 @@ AnthropicMessagesResponse, ) from vllm.entrypoints.anthropic.serving_messages import AnthropicServingMessages +from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.launcher import serve_http from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args @@ -88,7 +89,6 @@ cli_env_setup, load_aware_call, log_non_default_args, - process_chat_template, process_lora_modules, with_cancellation, ) @@ -1098,9 +1098,7 @@ async def init_app_state( supported_tasks = await engine_client.get_supported_tasks() logger.info("Supported tasks: %s", supported_tasks) - resolved_chat_template = await process_chat_template( - args.chat_template, engine_client, vllm_config.model_config - ) + resolved_chat_template = load_chat_template(args.chat_template) if args.tool_server == "demo": tool_server: ToolServer | None = DemoToolServer() diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index 3e421e21e3e8..cf0372c614b1 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -122,10 +122,7 @@ async def create_completion( try: lora_request = self._maybe_get_adapters(request) - if self.model_config.skip_tokenizer_init: - tokenizer = None - else: - tokenizer = await self.engine_client.get_tokenizer() + tokenizer = self.renderer.get_tokenizer() renderer = self._get_renderer(tokenizer) engine_prompts = await renderer.render_prompt_and_embeds( diff --git a/vllm/entrypoints/pooling/embed/serving.py b/vllm/entrypoints/pooling/embed/serving.py index 6675b1fc8793..5e791e963c5f 100644 --- a/vllm/entrypoints/pooling/embed/serving.py +++ b/vllm/entrypoints/pooling/embed/serving.py @@ -70,7 +70,7 @@ def __init__(self, *args, **kwargs): else None ) - @override + @override # type: ignore[misc] async def _preprocess( self, ctx: ServeContext, @@ -95,7 +95,7 @@ async def _preprocess( add_special_tokens=ctx.request.add_special_tokens, ) else: - tokenizer = await self.engine_client.get_tokenizer() + tokenizer = self.renderer.get_tokenizer() renderer = self._get_renderer(tokenizer) ctx.engine_prompts = await renderer.render_prompt( prompt_or_prompts=ctx.request.input, @@ -119,7 +119,7 @@ def _build_render_config(self, request: EmbeddingCompletionRequest) -> RenderCon add_special_tokens=request.add_special_tokens, ) - @override + @override # type: ignore[misc] def _build_response( self, ctx: ServeContext, @@ -362,7 +362,7 @@ async def _create_single_prompt_generator( priority=getattr(ctx.request, "priority", 0), ) - @override + @override # type: ignore[misc] async def _prepare_generators( self, ctx: ServeContext, @@ -432,7 +432,7 @@ async def _prepare_generators( # TODO: Use a vllm-specific Validation Error return self.create_error_response(str(e)) - @override + @override # type: ignore[misc] async def _collect_batch( self, ctx: ServeContext, @@ -665,7 +665,7 @@ async def create_embedding( return await super().handle(ctx) # type: ignore - @override + @override # type: ignore[misc] def _create_pooling_params( self, ctx: ServeContext[EmbeddingRequest], diff --git a/vllm/entrypoints/pooling/pooling/serving.py b/vllm/entrypoints/pooling/pooling/serving.py index d256d4c2178f..34ee93e93265 100644 --- a/vllm/entrypoints/pooling/pooling/serving.py +++ b/vllm/entrypoints/pooling/pooling/serving.py @@ -148,10 +148,7 @@ async def create_pooling( add_special_tokens=request.add_special_tokens, ) elif isinstance(request, PoolingCompletionRequest): - if self.model_config.skip_tokenizer_init: - tokenizer = None - else: - tokenizer = await self.engine_client.get_tokenizer() + tokenizer = self.renderer.get_tokenizer() renderer = self._get_renderer(tokenizer) engine_prompts = await renderer.render_prompt( prompt_or_prompts=request.input, diff --git a/vllm/entrypoints/pooling/score/serving.py b/vllm/entrypoints/pooling/score/serving.py index 846944948083..bec4510ce16b 100644 --- a/vllm/entrypoints/pooling/score/serving.py +++ b/vllm/entrypoints/pooling/score/serving.py @@ -283,8 +283,7 @@ async def _run_scoring( raw_request: Request | None = None, ) -> list[PoolingRequestOutput] | ErrorResponse: lora_request = self._maybe_get_adapters(request) - - tokenizer = await self.engine_client.get_tokenizer() + tokenizer = self.renderer.get_tokenizer() truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None) diff --git a/vllm/entrypoints/serve/tokenize/serving.py b/vllm/entrypoints/serve/tokenize/serving.py index a1293c82c355..6233e2e68c49 100644 --- a/vllm/entrypoints/serve/tokenize/serving.py +++ b/vllm/entrypoints/serve/tokenize/serving.py @@ -94,7 +94,7 @@ async def create_tokenize( add_special_tokens=request.add_special_tokens, ) else: - tokenizer = await self.engine_client.get_tokenizer() + tokenizer = self.renderer.get_tokenizer() renderer = self._get_renderer(tokenizer) engine_prompts = await renderer.render_prompt( prompt_or_prompts=request.prompt, @@ -136,8 +136,7 @@ async def create_detokenize( request_id = f"tokn-{self._base_request_id(raw_request)}" lora_request = self._maybe_get_adapters(request) - - tokenizer = await self.engine_client.get_tokenizer() + tokenizer = self.renderer.get_tokenizer() self._log_inputs( request_id, request.tokens, params=None, lora_request=lora_request @@ -157,7 +156,7 @@ async def get_tokenizer_info( ) -> TokenizerInfoResponse | ErrorResponse: """Get comprehensive tokenizer information.""" try: - tokenizer = await self.engine_client.get_tokenizer() + tokenizer = self.renderer.get_tokenizer() info = TokenizerInfo(tokenizer, self.chat_template).to_dict() return TokenizerInfoResponse(**info) except Exception as e: diff --git a/vllm/entrypoints/utils.py b/vllm/entrypoints/utils.py index f4a633c69cb0..949684d52814 100644 --- a/vllm/entrypoints/utils.py +++ b/vllm/entrypoints/utils.py @@ -6,21 +6,13 @@ import functools import os from argparse import Namespace -from pathlib import Path from typing import Any from fastapi import Request from fastapi.responses import JSONResponse, StreamingResponse from starlette.background import BackgroundTask, BackgroundTasks -from vllm.config import ModelConfig from vllm.engine.arg_utils import EngineArgs -from vllm.engine.protocol import EngineClient -from vllm.entrypoints.chat_utils import ( - load_chat_template, - resolve_hf_chat_template, - resolve_mistral_chat_template, -) from vllm.entrypoints.openai.cli_args import make_arg_parser from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, @@ -30,7 +22,6 @@ from vllm.entrypoints.openai.serving_models import LoRAModulePath from vllm.logger import init_logger from vllm.platforms import current_platform -from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.argparse_utils import FlexibleArgumentParser logger = init_logger(__name__) @@ -283,37 +274,3 @@ def process_lora_modules( else: lora_modules += default_mm_lora_paths return lora_modules - - -async def process_chat_template( - args_chat_template: Path | str | None, - engine_client: EngineClient, - model_config: ModelConfig, -) -> str | None: - resolved_chat_template = load_chat_template(args_chat_template) - if resolved_chat_template is not None: - # Get the tokenizer to check official template - tokenizer = await engine_client.get_tokenizer() - - if isinstance(tokenizer, MistralTokenizer): - # The warning is logged in resolve_mistral_chat_template. - resolved_chat_template = resolve_mistral_chat_template( - chat_template=resolved_chat_template - ) - else: - hf_chat_template = resolve_hf_chat_template( - tokenizer=tokenizer, - chat_template=None, - tools=None, - model_config=model_config, - ) - - if hf_chat_template != resolved_chat_template: - logger.warning( - "Using supplied chat template: %s\n" - "It is different from official chat template '%s'. " - "This discrepancy may lead to performance degradation.", - resolved_chat_template, - model_config.model, - ) - return resolved_chat_template diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index cd7ae5b6072d..e50fc58fda17 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -709,7 +709,7 @@ def tokenizer(self) -> TokenizerLike | None: def tokenizer(self, tokenizer: TokenizerLike | None) -> None: self.input_processor.tokenizer = tokenizer - async def get_tokenizer(self) -> TokenizerLike: + def get_tokenizer(self) -> TokenizerLike: return self.input_processor.get_tokenizer() @property From 4c81f01b9a07f39ced11c044c869abd65d856354 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 08:44:59 +0000 Subject: [PATCH 03/65] Move simplify Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_completion.py | 7 +++---- vllm/entrypoints/openai/serving_engine.py | 4 ++-- vllm/entrypoints/pooling/classify/serving.py | 2 +- vllm/entrypoints/pooling/embed/serving.py | 3 +-- vllm/entrypoints/pooling/pooling/serving.py | 3 +-- vllm/entrypoints/serve/tokenize/serving.py | 4 ++-- 6 files changed, 10 insertions(+), 13 deletions(-) diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index cf0372c614b1..d6b20b3f407f 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -121,10 +121,7 @@ async def create_completion( try: lora_request = self._maybe_get_adapters(request) - - tokenizer = self.renderer.get_tokenizer() - renderer = self._get_renderer(tokenizer) - + renderer = self._get_completion_renderer() engine_prompts = await renderer.render_prompt_and_embeds( prompt_or_prompts=request.prompt, prompt_embeds=request.prompt_embeds, @@ -251,6 +248,8 @@ async def create_completion( stream = request.stream and not request.use_beam_search # Streaming response + tokenizer = self.renderer.get_tokenizer() + if stream: return self.completion_stream_generator( request, diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 2bc89c4d0d24..2b21b2d47308 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -535,14 +535,14 @@ async def beam_search( prompt_logprobs=None, ) - def _get_renderer(self, tokenizer: TokenizerLike | None) -> BaseRenderer: + def _get_completion_renderer(self) -> BaseRenderer: """ Get a Renderer instance with the provided tokenizer. Uses shared async tokenizer pool for efficiency. """ return CompletionRenderer( model_config=self.model_config, - tokenizer=tokenizer, + tokenizer=self.renderer.tokenizer, async_tokenizer_pool=self._async_tokenizer_pool, ) diff --git a/vllm/entrypoints/pooling/classify/serving.py b/vllm/entrypoints/pooling/classify/serving.py index 377d037b2c10..5e6d1e7781e5 100644 --- a/vllm/entrypoints/pooling/classify/serving.py +++ b/vllm/entrypoints/pooling/classify/serving.py @@ -104,7 +104,7 @@ async def _preprocess( ctx.engine_prompts = [] return None - renderer = self._get_renderer(self.renderer.tokenizer) + renderer = self._get_completion_renderer() prompt_input = cast(str | list[str], input_data) ctx.engine_prompts = await renderer.render_prompt( prompt_or_prompts=prompt_input, diff --git a/vllm/entrypoints/pooling/embed/serving.py b/vllm/entrypoints/pooling/embed/serving.py index 5e791e963c5f..8d10be5f5a9a 100644 --- a/vllm/entrypoints/pooling/embed/serving.py +++ b/vllm/entrypoints/pooling/embed/serving.py @@ -95,8 +95,7 @@ async def _preprocess( add_special_tokens=ctx.request.add_special_tokens, ) else: - tokenizer = self.renderer.get_tokenizer() - renderer = self._get_renderer(tokenizer) + renderer = self._get_completion_renderer() ctx.engine_prompts = await renderer.render_prompt( prompt_or_prompts=ctx.request.input, config=self._build_render_config(ctx.request), diff --git a/vllm/entrypoints/pooling/pooling/serving.py b/vllm/entrypoints/pooling/pooling/serving.py index 34ee93e93265..44c27fd8522e 100644 --- a/vllm/entrypoints/pooling/pooling/serving.py +++ b/vllm/entrypoints/pooling/pooling/serving.py @@ -148,8 +148,7 @@ async def create_pooling( add_special_tokens=request.add_special_tokens, ) elif isinstance(request, PoolingCompletionRequest): - tokenizer = self.renderer.get_tokenizer() - renderer = self._get_renderer(tokenizer) + renderer = self._get_completion_renderer() engine_prompts = await renderer.render_prompt( prompt_or_prompts=request.input, config=self._build_render_config(request), diff --git a/vllm/entrypoints/serve/tokenize/serving.py b/vllm/entrypoints/serve/tokenize/serving.py index 6233e2e68c49..29eb2ec609e6 100644 --- a/vllm/entrypoints/serve/tokenize/serving.py +++ b/vllm/entrypoints/serve/tokenize/serving.py @@ -94,8 +94,7 @@ async def create_tokenize( add_special_tokens=request.add_special_tokens, ) else: - tokenizer = self.renderer.get_tokenizer() - renderer = self._get_renderer(tokenizer) + renderer = self._get_completion_renderer() engine_prompts = await renderer.render_prompt( prompt_or_prompts=request.prompt, config=self._build_render_config(request), @@ -115,6 +114,7 @@ async def create_tokenize( token_strs = None if request.return_token_strs: + tokenizer = self.renderer.get_tokenizer() token_strs = tokenizer.convert_ids_to_tokens(input_ids) return TokenizeResponse( From 4a37cda80140e8c9b0ea0436243846e34c020113 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 08:45:29 +0000 Subject: [PATCH 04/65] Typo Signed-off-by: DarkLight1337 --- vllm/entrypoints/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 7725fae16dd9..53ca52cb2abd 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -816,7 +816,7 @@ def preprocess_chat( prompts = list[TextPrompt | TokensPrompt]() for msgs in list_of_messages: - # NOTE: parse_mesrender_messagessages() currently doesn't + # NOTE: renderer.render_messages() currently doesn't # handle mm_processor_kwargs, since there is no implementation in # the chat message parsing for it. _, prompt = renderer.render_messages( From 3d11a96974985e4544a887afe96363d87335520e Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 08:47:47 +0000 Subject: [PATCH 05/65] Set the tokenizer name Signed-off-by: DarkLight1337 --- vllm/renderers/registry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 376d9e3ab6f8..591db9654d49 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -98,6 +98,7 @@ def renderer_from_config(config: "ModelConfig"): tokenizer_mode = "hf" tokenizer_kwargs = dict[str, Any]( + tokenizer_name=tokenizer_name, trust_remote_code=trust_remote_code, revision=tokenizer_revision, **tokenizer_kwargs, From e66ebd825c2d59ea57978d0c25b4f144d3c2efcf Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 08:50:05 +0000 Subject: [PATCH 06/65] Simplify Signed-off-by: DarkLight1337 --- vllm/entrypoints/context.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/context.py b/vllm/entrypoints/context.py index 25aca81d619b..6c404b0ebb66 100644 --- a/vllm/entrypoints/context.py +++ b/vllm/entrypoints/context.py @@ -247,8 +247,9 @@ def __init__( if reasoning_parser_cls is None: raise ValueError("reasoning_parser_cls must be provided.") + tokenizer = renderer.get_tokenizer() self.parser = get_responses_parser_for_simple_context( - tokenizer=renderer.get_tokenizer(), + tokenizer=tokenizer, reasoning_parser_cls=reasoning_parser_cls, response_messages=response_messages, request=request, @@ -257,7 +258,7 @@ def __init__( self.tool_parser_cls = tool_parser_cls self.request = request self.renderer = renderer - self.tokenizer = renderer.get_tokenizer() + self.tokenizer = tokenizer self.available_tools = available_tools or [] self._tool_sessions: dict[str, ClientSession | Tool] = {} From 02e64f581de43218449b3c3450896c43a89f40e0 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 09:11:12 +0000 Subject: [PATCH 07/65] Fix init Signed-off-by: DarkLight1337 --- vllm/renderers/hf.py | 5 +- vllm/renderers/registry.py | 114 ++++++++++++-------------------- vllm/tokenizers/registry.py | 127 ++++++++++++++++++++++++++---------- 3 files changed, 137 insertions(+), 109 deletions(-) diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index dec21c031bee..b5c2e58dd41b 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -26,7 +26,7 @@ from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger from vllm.tokenizers import get_tokenizer -from vllm.tokenizers.hf import HfTokenizer +from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path from vllm.transformers_utils.processor import cached_get_processor from vllm.utils.func_utils import supports_kw @@ -497,7 +497,8 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = get_tokenizer(HfTokenizer, **tokenizer_kwargs) + tokenizer = get_tokenizer(CachedHfTokenizer, **tokenizer_kwargs) + assert isinstance(tokenizer, HfTokenizer) self._tokenizer = tokenizer diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 591db9654d49..c40e3ce0a15c 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -1,12 +1,9 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import importlib.util -from typing import TYPE_CHECKING, Any, TypeVar - -from typing_extensions import assert_never +from typing import TYPE_CHECKING, Any from vllm.logger import init_logger -from vllm.transformers_utils.repo_utils import list_filtered_repo_files +from vllm.tokenizers.registry import tokenizer_mode_kwargs_from_config from vllm.utils.import_utils import resolve_obj_by_qualname from .protocol import RendererLike @@ -16,21 +13,16 @@ logger = init_logger(__name__) -_T = TypeVar("_T", bound=type[RendererLike]) - class RendererRegistry: - # Renderer name -> (renderer module, renderer class) - REGISTRY: dict[str, tuple[str, str]] = { - "deepseekv32": ("vllm.renderers.deepseekv32", "DeepseekV32Renderer"), - "hf": ("vllm.renderers.hf", "HfRenderer"), - "mistral": ("vllm.renderers.mistral", "MistralRenderer"), - "terratorch": ("vllm.renderers.terratorch", "TerratorchRenderer"), - } + def __init__(self) -> None: + super().__init__() + + # Renderer name -> (renderer module, renderer class) + self._registry: dict[str, tuple[str, str]] = {} - @staticmethod - def register(renderer_mode: str, module: str, class_name: str) -> None: - if renderer_mode in RendererRegistry.REGISTRY: + def register(self, renderer_mode: str, module: str, class_name: str) -> None: + if renderer_mode in self._registry: logger.warning( "%s.%s is already registered for renderer_mode=%r. " "It is overwritten by the new one.", @@ -39,73 +31,49 @@ def register(renderer_mode: str, module: str, class_name: str) -> None: renderer_mode, ) - RendererRegistry.REGISTRY[renderer_mode] = (module, class_name) + self._registry[renderer_mode] = (module, class_name) return None - @staticmethod + def load_renderer_cls(self, renderer_mode: str) -> type[RendererLike]: + if renderer_mode not in self._registry: + raise ValueError(f"No renderer registered for {renderer_mode=!r}.") + + module, class_name = self._registry[renderer_mode] + logger.debug_once(f"Loading {class_name} for {renderer_mode=!r}") + + return resolve_obj_by_qualname(f"{module}.{class_name}") + def init_renderer( + self, renderer_mode: str, config: "ModelConfig", tokenizer_kwargs: dict[str, Any], ) -> RendererLike: - if renderer_mode not in RendererRegistry.REGISTRY: - raise ValueError(f"No renderer registered for {renderer_mode=!r}.") + renderer_cls = self.load_renderer_cls(renderer_mode) + return renderer_cls.from_config(config, tokenizer_kwargs) - module, class_name = RendererRegistry.REGISTRY[renderer_mode] - logger.debug_once(f"Loading {class_name} for {renderer_mode=!r}") - cls_: type[RendererLike] = resolve_obj_by_qualname(f"{module}.{class_name}") - return cls_.from_config(config, tokenizer_kwargs) +RENDERER_REGISTRY = RendererRegistry() +"""The global `RendererRegistry` instance.""" +RENDERER_REGISTRY._registry.update( + { + "deepseekv32": ("vllm.renderers.deepseekv32", "DeepseekV32Renderer"), + "hf": ("vllm.renderers.hf", "HfRenderer"), + "mistral": ("vllm.renderers.mistral", "MistralRenderer"), + "terratorch": ("vllm.renderers.terratorch", "TerratorchRenderer"), + } +) -def renderer_from_config(config: "ModelConfig"): - tokenizer_name = config.tokenizer - tokenizer_mode = config.tokenizer_mode - tokenizer_revision = config.tokenizer_revision - trust_remote_code = config.trust_remote_code - tokenizer_kwargs = dict[str, Any]() - runner_type = config.runner_type - if runner_type == "generate" or runner_type == "draft": - tokenizer_kwargs["truncation_side"] = "left" - elif runner_type == "pooling": - tokenizer_kwargs["truncation_side"] = "right" +def renderer_from_config(config: "ModelConfig", **kwargs): + tokenizer_mode, tokenizer_kwargs = tokenizer_mode_kwargs_from_config(config) + tokenizer_kwargs.update(kwargs) + + if config.tokenizer_mode == "auto" and config.model_impl == "terratorch": + renderer_mode = "terratorch" else: - assert_never(runner_type) - - tokenizer_mode = config.tokenizer_mode - if tokenizer_mode == "slow": - if tokenizer_kwargs.get("use_fast", False): - raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") - - tokenizer_mode = "hf" - tokenizer_kwargs["use_fast"] = False - - # Try to use official Mistral tokenizer if possible - if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"): - allow_patterns = ["tekken.json", "tokenizer.model.v*"] - files_list = list_filtered_repo_files( - model_name_or_path=str(tokenizer_name), - allow_patterns=allow_patterns, - revision=tokenizer_revision, - ) - if len(files_list) > 0: - tokenizer_mode = "mistral" - - # Fallback to HF tokenizer - if tokenizer_mode == "auto": - tokenizer_mode = "hf" - - tokenizer_kwargs = dict[str, Any]( - tokenizer_name=tokenizer_name, - trust_remote_code=trust_remote_code, - revision=tokenizer_revision, - **tokenizer_kwargs, - ) - - return RendererRegistry.init_renderer( - tokenizer_mode, - config, - tokenizer_kwargs, - ) + renderer_mode = tokenizer_mode + + return RENDERER_REGISTRY.init_renderer(renderer_mode, config, tokenizer_kwargs) diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 35ac00f69f5c..8ca24b49757c 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -1,11 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import importlib.util from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING, Any, TypeVar import huggingface_hub -from typing_extensions import assert_never +from typing_extensions import assert_never, deprecated import vllm.envs as envs from vllm.logger import init_logger @@ -16,6 +17,7 @@ is_remote_gguf, split_remote_gguf, ) +from vllm.transformers_utils.repo_utils import list_filtered_repo_files from vllm.utils.import_utils import resolve_obj_by_qualname from .protocol import TokenizerLike @@ -29,16 +31,14 @@ class TokenizerRegistry: - # Tokenizer name -> (tokenizer module, tokenizer class) - REGISTRY: dict[str, tuple[str, str]] = { - "deepseekv32": ("vllm.tokenizers.deepseekv32", "DeepseekV32Tokenizer"), - "hf": ("vllm.tokenizers.hf", "CachedHfTokenizer"), - "mistral": ("vllm.tokenizers.mistral", "MistralTokenizer"), - } + def __init__(self) -> None: + super().__init__() + + # Tokenizer name -> (tokenizer module, tokenizer class) + self._registry: dict[str, tuple[str, str]] = {} - @staticmethod - def register(tokenizer_mode: str, module: str, class_name: str) -> None: - if tokenizer_mode in TokenizerRegistry.REGISTRY: + def register(self, tokenizer_mode: str, module: str, class_name: str) -> None: + if tokenizer_mode in self._registry: logger.warning( "%s.%s is already registered for tokenizer_mode=%r. " "It is overwritten by the new one.", @@ -47,20 +47,34 @@ def register(tokenizer_mode: str, module: str, class_name: str) -> None: tokenizer_mode, ) - TokenizerRegistry.REGISTRY[tokenizer_mode] = (module, class_name) + self._registry[tokenizer_mode] = (module, class_name) return None - @staticmethod - def init_tokenizer(tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: - if tokenizer_mode not in TokenizerRegistry.REGISTRY: + def load_tokenizer_cls(self, tokenizer_mode: str) -> type[TokenizerLike]: + if tokenizer_mode not in self._registry: raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.") - module, class_name = TokenizerRegistry.REGISTRY[tokenizer_mode] + module, class_name = self._registry[tokenizer_mode] logger.debug_once(f"Loading {class_name} for {tokenizer_mode=!r}") - cls_: type[TokenizerLike] = resolve_obj_by_qualname(f"{module}.{class_name}") - return cls_.from_pretrained(*args, **kwargs) + return resolve_obj_by_qualname(f"{module}.{class_name}") + + def init_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: + tokenizer_cls = self.load_tokenizer_cls(tokenizer_mode) + return tokenizer_cls.from_pretrained(*args, **kwargs) + + +TOKENIZER_REGISTRY = TokenizerRegistry() +"""The global `TokenizerRegistry` instance.""" + +TOKENIZER_REGISTRY._registry.update( + { + "deepseekv32": ("vllm.tokenizers.deepseekv32", "DeepseekV32Tokenizer"), + "hf": ("vllm.tokenizers.hf", "CachedHfTokenizer"), + "mistral": ("vllm.tokenizers.mistral", "MistralTokenizer"), + } +) def get_tokenizer( @@ -129,29 +143,74 @@ def get_tokenizer( return tokenizer # type: ignore +def tokenizer_mode_kwargs_from_config(config: "ModelConfig"): + tokenizer_name = config.tokenizer + tokenizer_mode = config.tokenizer_mode + tokenizer_revision = config.tokenizer_revision + trust_remote_code = config.trust_remote_code + tokenizer_kwargs = dict[str, Any]() + + runner_type = config.runner_type + if runner_type == "generate" or runner_type == "draft": + tokenizer_kwargs["truncation_side"] = "left" + elif runner_type == "pooling": + tokenizer_kwargs["truncation_side"] = "right" + else: + assert_never(runner_type) + + tokenizer_mode = config.tokenizer_mode + if tokenizer_mode == "slow": + if tokenizer_kwargs.get("use_fast", False): + raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") + + tokenizer_mode = "hf" + tokenizer_kwargs["use_fast"] = False + + # Try to use official Mistral tokenizer if possible + if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"): + allow_patterns = ["tekken.json", "tokenizer.model.v*"] + files_list = list_filtered_repo_files( + model_name_or_path=str(tokenizer_name), + allow_patterns=allow_patterns, + revision=tokenizer_revision, + ) + if len(files_list) > 0: + tokenizer_mode = "mistral" + + # Fallback to HF tokenizer + if tokenizer_mode == "auto": + tokenizer_mode = "hf" + + tokenizer_kwargs = { + "tokenizer_name": tokenizer_name, + "trust_remote_code": trust_remote_code, + "revision": tokenizer_revision, + **tokenizer_kwargs, + } + + return tokenizer_mode, tokenizer_kwargs + + cached_get_tokenizer = lru_cache(get_tokenizer) def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs): + if model_config.skip_tokenizer_init: + return None + + tokenizer_mode, tokenizer_kwargs = tokenizer_mode_kwargs_from_config(model_config) + tokenizer_kwargs.update(kwargs) + + tokenizer_cls = TOKENIZER_REGISTRY.load_tokenizer_cls(tokenizer_mode) + return cached_get_tokenizer( - model_config.tokenizer, - tokenizer_mode=model_config.tokenizer_mode, - revision=model_config.tokenizer_revision, - trust_remote_code=model_config.trust_remote_code, - **kwargs, + tokenizer_cls, # type: ignore[arg-type] + **tokenizer_kwargs, ) +@deprecated( + "Renamed to `cached_tokenizer_from_config`. The old name will be removed in v0.14." +) def init_tokenizer_from_config(model_config: "ModelConfig"): - if model_config.skip_tokenizer_init: - return None - - runner_type = model_config.runner_type - if runner_type == "generate" or runner_type == "draft": - truncation_side = "left" - elif runner_type == "pooling": - truncation_side = "right" - else: - assert_never(runner_type) - - return cached_tokenizer_from_config(model_config, truncation_side=truncation_side) + return cached_tokenizer_from_config(model_config) From d8177a1d413bf5d51ba09c7c7a24eee109f56907 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 09:49:27 +0000 Subject: [PATCH 08/65] Reuse code Signed-off-by: DarkLight1337 --- tests/tokenizers_/test_registry.py | 2 +- vllm/renderers/deepseekv32.py | 4 +- vllm/renderers/hf.py | 4 +- vllm/renderers/mistral.py | 4 +- vllm/renderers/registry.py | 11 ++- vllm/tokenizers/__init__.py | 2 + vllm/tokenizers/registry.py | 150 +++++++++++++++++++---------- 7 files changed, 114 insertions(+), 63 deletions(-) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index f2001bc19a37..3a2f84ea976b 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -43,7 +43,7 @@ def is_fast(self) -> bool: def test_customized_tokenizer(): TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__) - tokenizer = TokenizerRegistry.init_tokenizer("test_tokenizer", "abc") + tokenizer = TokenizerRegistry.load_tokenizer("test_tokenizer", "abc") assert isinstance(tokenizer, TestTokenizer) assert tokenizer.path_or_repo_id == "abc" assert tokenizer.bos_token_id == 0 diff --git a/vllm/renderers/deepseekv32.py b/vllm/renderers/deepseekv32.py index 40878af9f262..2cc7fd01dda8 100644 --- a/vllm/renderers/deepseekv32.py +++ b/vllm/renderers/deepseekv32.py @@ -11,7 +11,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import get_tokenizer +from vllm.tokenizers import init_tokenizer from vllm.tokenizers.deepseekv32 import DeepseekV32Tokenizer from .protocol import RendererLike @@ -40,7 +40,7 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = get_tokenizer(DeepseekV32Tokenizer, **tokenizer_kwargs) + tokenizer = init_tokenizer(DeepseekV32Tokenizer, **tokenizer_kwargs) self._tokenizer = tokenizer diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index b5c2e58dd41b..ba9330db2811 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -25,7 +25,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import get_tokenizer +from vllm.tokenizers import init_tokenizer from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path from vllm.transformers_utils.processor import cached_get_processor @@ -497,7 +497,7 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = get_tokenizer(CachedHfTokenizer, **tokenizer_kwargs) + tokenizer = init_tokenizer(CachedHfTokenizer, **tokenizer_kwargs) assert isinstance(tokenizer, HfTokenizer) self._tokenizer = tokenizer diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index 5268608674c2..b17e2197d6dd 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -12,7 +12,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import get_tokenizer +from vllm.tokenizers import init_tokenizer from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.async_utils import make_async @@ -69,7 +69,7 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = get_tokenizer(MistralTokenizer, **tokenizer_kwargs) + tokenizer = init_tokenizer(MistralTokenizer, **tokenizer_kwargs) self._tokenizer = tokenizer diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index c40e3ce0a15c..39e2b54d70e3 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any from vllm.logger import init_logger -from vllm.tokenizers.registry import tokenizer_mode_kwargs_from_config +from vllm.tokenizers.registry import tokenizer_args_from_config from vllm.utils.import_utils import resolve_obj_by_qualname from .protocol import RendererLike @@ -44,7 +44,7 @@ def load_renderer_cls(self, renderer_mode: str) -> type[RendererLike]: return resolve_obj_by_qualname(f"{module}.{class_name}") - def init_renderer( + def load_renderer( self, renderer_mode: str, config: "ModelConfig", @@ -68,12 +68,15 @@ def init_renderer( def renderer_from_config(config: "ModelConfig", **kwargs): - tokenizer_mode, tokenizer_kwargs = tokenizer_mode_kwargs_from_config(config) + tokenizer_mode, tokenizer_args, tokenizer_kwargs = tokenizer_args_from_config( + config, **kwargs + ) tokenizer_kwargs.update(kwargs) + tokenizer_kwargs["tokenizer_name"] = tokenizer_args[0] if config.tokenizer_mode == "auto" and config.model_impl == "terratorch": renderer_mode = "terratorch" else: renderer_mode = tokenizer_mode - return RENDERER_REGISTRY.init_renderer(renderer_mode, config, tokenizer_kwargs) + return RENDERER_REGISTRY.load_renderer(renderer_mode, config, tokenizer_kwargs) diff --git a/vllm/tokenizers/__init__.py b/vllm/tokenizers/__init__.py index 31e74b1a16e2..8f9403a77cdf 100644 --- a/vllm/tokenizers/__init__.py +++ b/vllm/tokenizers/__init__.py @@ -7,6 +7,7 @@ cached_get_tokenizer, cached_tokenizer_from_config, get_tokenizer, + init_tokenizer, init_tokenizer_from_config, ) @@ -16,5 +17,6 @@ "cached_get_tokenizer", "get_tokenizer", "cached_tokenizer_from_config", + "init_tokenizer", "init_tokenizer_from_config", ] diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 8ca24b49757c..d3a44bcb39f2 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -3,7 +3,7 @@ import importlib.util from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Any, TypeVar +from typing import TYPE_CHECKING, Literal, TypeVar import huggingface_hub from typing_extensions import assert_never, deprecated @@ -27,10 +27,8 @@ logger = init_logger(__name__) -_T = TypeVar("_T", bound=TokenizerLike) - -class TokenizerRegistry: +class TokenizerRegistryType: def __init__(self) -> None: super().__init__() @@ -60,15 +58,15 @@ def load_tokenizer_cls(self, tokenizer_mode: str) -> type[TokenizerLike]: return resolve_obj_by_qualname(f"{module}.{class_name}") - def init_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: + def load_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: tokenizer_cls = self.load_tokenizer_cls(tokenizer_mode) return tokenizer_cls.from_pretrained(*args, **kwargs) -TOKENIZER_REGISTRY = TokenizerRegistry() -"""The global `TokenizerRegistry` instance.""" +TokenizerRegistry = TokenizerRegistryType() +"""The global `TokenizerRegistryType` instance.""" -TOKENIZER_REGISTRY._registry.update( +TokenizerRegistry._registry.update( { "deepseekv32": ("vllm.tokenizers.deepseekv32", "DeepseekV32Tokenizer"), "hf": ("vllm.tokenizers.hf", "CachedHfTokenizer"), @@ -77,16 +75,18 @@ def init_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: ) -def get_tokenizer( - tokenizer_cls: type[_T], +def resolve_tokenizer_args( tokenizer_name: str | Path, *args, + runner_type: Literal["generate", "pooling"] = "generate", + tokenizer_mode: str = "auto", trust_remote_code: bool = False, revision: str | None = None, download_dir: str | None = None, **kwargs, -) -> _T: - """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" +): + tokenizer_kwargs = kwargs + if envs.VLLM_USE_MODELSCOPE: # download model from ModelScope hub, # lazy import so that modelscope is not required for normal use. @@ -113,7 +113,7 @@ def get_tokenizer( # Separate model folder from file path for GGUF models if is_gguf(tokenizer_name): if check_gguf_file(tokenizer_name): - kwargs["gguf_file"] = Path(tokenizer_name).name + tokenizer_kwargs["gguf_file"] = Path(tokenizer_name).name tokenizer_name = Path(tokenizer_name).parent elif is_remote_gguf(tokenizer_name): tokenizer_name, quant_type = split_remote_gguf(tokenizer_name) @@ -123,34 +123,8 @@ def get_tokenizer( quant_type, revision=revision, ) - kwargs["gguf_file"] = gguf_file - - tokenizer_args = (tokenizer_name, *args) - tokenizer_kwargs = dict[str, Any]( - trust_remote_code=trust_remote_code, - revision=revision, - download_dir=download_dir, - **kwargs, - ) - - tokenizer = tokenizer_cls.from_pretrained(*tokenizer_args, **tokenizer_kwargs) - if not tokenizer.is_fast: - logger.warning( - "Using a slow tokenizer. This might cause a significant " - "slowdown. Consider using a fast tokenizer instead." - ) - - return tokenizer # type: ignore + tokenizer_kwargs["gguf_file"] = gguf_file - -def tokenizer_mode_kwargs_from_config(config: "ModelConfig"): - tokenizer_name = config.tokenizer - tokenizer_mode = config.tokenizer_mode - tokenizer_revision = config.tokenizer_revision - trust_remote_code = config.trust_remote_code - tokenizer_kwargs = dict[str, Any]() - - runner_type = config.runner_type if runner_type == "generate" or runner_type == "draft": tokenizer_kwargs["truncation_side"] = "left" elif runner_type == "pooling": @@ -158,7 +132,6 @@ def tokenizer_mode_kwargs_from_config(config: "ModelConfig"): else: assert_never(runner_type) - tokenizer_mode = config.tokenizer_mode if tokenizer_mode == "slow": if tokenizer_kwargs.get("use_fast", False): raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") @@ -172,7 +145,7 @@ def tokenizer_mode_kwargs_from_config(config: "ModelConfig"): files_list = list_filtered_repo_files( model_name_or_path=str(tokenizer_name), allow_patterns=allow_patterns, - revision=tokenizer_revision, + revision=revision, ) if len(files_list) > 0: tokenizer_mode = "mistral" @@ -181,14 +154,58 @@ def tokenizer_mode_kwargs_from_config(config: "ModelConfig"): if tokenizer_mode == "auto": tokenizer_mode = "hf" + tokenizer_args = (tokenizer_name, *args) tokenizer_kwargs = { - "tokenizer_name": tokenizer_name, "trust_remote_code": trust_remote_code, - "revision": tokenizer_revision, + "revision": revision, **tokenizer_kwargs, } - return tokenizer_mode, tokenizer_kwargs + return tokenizer_mode, tokenizer_args, tokenizer_kwargs + + +cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args) + + +def tokenizer_args_from_config(config: "ModelConfig", **kwargs): + return cached_resolve_tokenizer_args( + config.tokenizer, + runner_type=config.runner_type, + tokenizer_mode=config.tokenizer_mode, + revision=config.tokenizer_revision, + trust_remote_code=config.trust_remote_code, + **kwargs, + ) + + +def get_tokenizer( + tokenizer_name: str | Path, + *args, + trust_remote_code: bool = False, + revision: str | None = None, + download_dir: str | None = None, + **kwargs, +) -> TokenizerLike: + """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" + tokenizer_mode, tokenizer_args, tokenizer_kwargs = cached_resolve_tokenizer_args( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + download_dir=download_dir, + **kwargs, + ) + tokenizer_kwargs.update(kwargs) + + tokenizer_cls = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode) + tokenizer = tokenizer_cls.from_pretrained(*tokenizer_args, **tokenizer_kwargs) + if not tokenizer.is_fast: + logger.warning( + "Using a slow tokenizer. This might cause a significant " + "slowdown. Consider using a fast tokenizer instead." + ) + + return tokenizer # type: ignore cached_get_tokenizer = lru_cache(get_tokenizer) @@ -198,14 +215,13 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs): if model_config.skip_tokenizer_init: return None - tokenizer_mode, tokenizer_kwargs = tokenizer_mode_kwargs_from_config(model_config) - tokenizer_kwargs.update(kwargs) - - tokenizer_cls = TOKENIZER_REGISTRY.load_tokenizer_cls(tokenizer_mode) - return cached_get_tokenizer( - tokenizer_cls, # type: ignore[arg-type] - **tokenizer_kwargs, + model_config.tokenizer, + runner_type=model_config.runner_type, + tokenizer_mode=model_config.tokenizer_mode, + revision=model_config.tokenizer_revision, + trust_remote_code=model_config.trust_remote_code, + **kwargs, ) @@ -214,3 +230,33 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs): ) def init_tokenizer_from_config(model_config: "ModelConfig"): return cached_tokenizer_from_config(model_config) + + +_T = TypeVar("_T", bound=TokenizerLike) + + +def init_tokenizer( + tokenizer_cls: type[_T], + tokenizer_name: str | Path, + trust_remote_code: bool = False, + revision: str | None = None, + download_dir: str | None = None, + **kwargs, +) -> _T: + tokenizer_mode, tokenizer_args, tokenizer_kwargs = cached_resolve_tokenizer_args( + tokenizer_name, + trust_remote_code=trust_remote_code, + revision=revision, + download_dir=download_dir, + **kwargs, + ) + tokenizer_kwargs.update(kwargs) + + tokenizer = tokenizer_cls.from_pretrained(*tokenizer_args, **tokenizer_kwargs) + if not tokenizer.is_fast: + logger.warning( + "Using a slow tokenizer. This might cause a significant " + "slowdown. Consider using a fast tokenizer instead." + ) + + return tokenizer # type: ignore From a00c1646e3496b5fb27d2426e2f6bb175cca9d67 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 09:50:03 +0000 Subject: [PATCH 09/65] Type checking Signed-off-by: DarkLight1337 --- tools/pre_commit/mypy.py | 1 + vllm/renderers/protocol.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 724b39304426..0c1146e1f746 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -35,6 +35,7 @@ "vllm/multimodal", "vllm/platforms", "vllm/plugins", + "vllm/renderers", "vllm/tokenizers", "vllm/transformers_utils", "vllm/triton_utils", diff --git a/vllm/renderers/protocol.py b/vllm/renderers/protocol.py index 641121f880bc..02f3161faffd 100644 --- a/vllm/renderers/protocol.py +++ b/vllm/renderers/protocol.py @@ -29,7 +29,7 @@ def tokenizer(self) -> TokenizerLike | None: # NOTE: Remove this once LLM.tokenizer.setter is removed @tokenizer.setter def tokenizer(self, tokenizer: TokenizerLike | None) -> None: - self._tokenizer = tokenizer + raise NotImplementedError def get_tokenizer(self) -> TokenizerLike: tokenizer = self.tokenizer From 8a931d7b7e18dd2152f4632ac79cd7cf063735fd Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 09:58:15 +0000 Subject: [PATCH 10/65] Cached Signed-off-by: DarkLight1337 --- vllm/renderers/deepseekv32.py | 4 ++-- vllm/renderers/hf.py | 4 ++-- vllm/renderers/mistral.py | 4 ++-- vllm/tokenizers/__init__.py | 2 ++ vllm/tokenizers/registry.py | 3 +++ 5 files changed, 11 insertions(+), 6 deletions(-) diff --git a/vllm/renderers/deepseekv32.py b/vllm/renderers/deepseekv32.py index 2cc7fd01dda8..e0f487a86efe 100644 --- a/vllm/renderers/deepseekv32.py +++ b/vllm/renderers/deepseekv32.py @@ -11,7 +11,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import init_tokenizer +from vllm.tokenizers import cached_init_tokenizer from vllm.tokenizers.deepseekv32 import DeepseekV32Tokenizer from .protocol import RendererLike @@ -40,7 +40,7 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = init_tokenizer(DeepseekV32Tokenizer, **tokenizer_kwargs) + tokenizer = cached_init_tokenizer(DeepseekV32Tokenizer, **tokenizer_kwargs) self._tokenizer = tokenizer diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index ba9330db2811..3e8aa08cd0f4 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -25,7 +25,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import init_tokenizer +from vllm.tokenizers import cached_init_tokenizer from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path from vllm.transformers_utils.processor import cached_get_processor @@ -497,7 +497,7 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = init_tokenizer(CachedHfTokenizer, **tokenizer_kwargs) + tokenizer = cached_init_tokenizer(CachedHfTokenizer, **tokenizer_kwargs) assert isinstance(tokenizer, HfTokenizer) self._tokenizer = tokenizer diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index b17e2197d6dd..1c39fc43d649 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -12,7 +12,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import init_tokenizer +from vllm.tokenizers import cached_init_tokenizer from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.async_utils import make_async @@ -69,7 +69,7 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = init_tokenizer(MistralTokenizer, **tokenizer_kwargs) + tokenizer = cached_init_tokenizer(MistralTokenizer, **tokenizer_kwargs) self._tokenizer = tokenizer diff --git a/vllm/tokenizers/__init__.py b/vllm/tokenizers/__init__.py index 8f9403a77cdf..a0afc174be9c 100644 --- a/vllm/tokenizers/__init__.py +++ b/vllm/tokenizers/__init__.py @@ -5,6 +5,7 @@ from .registry import ( TokenizerRegistry, cached_get_tokenizer, + cached_init_tokenizer, cached_tokenizer_from_config, get_tokenizer, init_tokenizer, @@ -17,6 +18,7 @@ "cached_get_tokenizer", "get_tokenizer", "cached_tokenizer_from_config", + "cached_init_tokenizer", "init_tokenizer", "init_tokenizer_from_config", ] diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index d3a44bcb39f2..4c50b5ddf24d 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -260,3 +260,6 @@ def init_tokenizer( ) return tokenizer # type: ignore + + +cached_init_tokenizer = lru_cache(init_tokenizer) From 6909fe7dafa8a35f99dc262974d3e765333d0f0e Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:02:44 +0000 Subject: [PATCH 11/65] Fix mypy Signed-off-by: DarkLight1337 --- vllm/renderers/deepseekv32.py | 20 +++++++++++--------- vllm/renderers/hf.py | 27 ++++++++++++++++----------- vllm/renderers/mistral.py | 20 +++++++++++--------- vllm/renderers/terratorch.py | 2 +- 4 files changed, 39 insertions(+), 30 deletions(-) diff --git a/vllm/renderers/deepseekv32.py b/vllm/renderers/deepseekv32.py index e0f487a86efe..f1204515ebcb 100644 --- a/vllm/renderers/deepseekv32.py +++ b/vllm/renderers/deepseekv32.py @@ -44,7 +44,7 @@ def __init__( self._tokenizer = tokenizer - @property + @property # type: ignore[override] def tokenizer(self) -> DeepseekV32Tokenizer | None: return self._tokenizer @@ -77,11 +77,12 @@ def render_messages( messages=messages, **kwargs, ) - if isinstance(prompt_raw, str): - prompt = TextPrompt(prompt=prompt_raw) - else: - prompt = TokensPrompt(prompt_token_ids=prompt_raw) + prompt = ( + TextPrompt(prompt=prompt_raw) + if isinstance(prompt_raw, str) + else TokensPrompt(prompt_token_ids=prompt_raw) + ) if mm_data is not None: prompt["multi_modal_data"] = mm_data if mm_uuids is not None: @@ -106,11 +107,12 @@ async def render_messages_async( messages=messages, **kwargs, ) - if isinstance(prompt_raw, str): - prompt = TextPrompt(prompt=prompt_raw) - else: - prompt = TokensPrompt(prompt_token_ids=prompt_raw) + prompt = ( + TextPrompt(prompt=prompt_raw) + if isinstance(prompt_raw, str) + else TokensPrompt(prompt_token_ids=prompt_raw) + ) if mm_data_future is not None: prompt["multi_modal_data"] = await mm_data_future if mm_uuids is not None: diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 3e8aa08cd0f4..9f39b0dc49dc 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -497,12 +497,15 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = cached_init_tokenizer(CachedHfTokenizer, **tokenizer_kwargs) + tokenizer = cached_init_tokenizer( + CachedHfTokenizer, # type: ignore[type-abstract] + **tokenizer_kwargs, + ) assert isinstance(tokenizer, HfTokenizer) self._tokenizer = tokenizer - @property + @property # type: ignore[override] def tokenizer(self) -> HfTokenizer | None: return self._tokenizer @@ -545,11 +548,12 @@ def render_messages( conversation, **kwargs, ) - if isinstance(prompt_raw, str): - prompt = TextPrompt(prompt=prompt_raw) - else: - prompt = TokensPrompt(prompt_token_ids=prompt_raw) + prompt = ( + TextPrompt(prompt=prompt_raw) + if isinstance(prompt_raw, str) + else TokensPrompt(prompt_token_ids=prompt_raw) + ) if mm_data is not None: prompt["multi_modal_data"] = mm_data if mm_uuids is not None: @@ -560,7 +564,7 @@ def render_messages( async def render_messages_async( self, messages: list[ChatCompletionMessageParam], - chat_template_content_format: ChatTemplateContentFormat, + chat_template_content_format: ChatTemplateContentFormatOption = "auto", **kwargs, ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: model_config = self.config @@ -584,11 +588,12 @@ async def render_messages_async( conversation, **kwargs, ) - if isinstance(prompt_raw, str): - prompt = TextPrompt(prompt=prompt_raw) - else: - prompt = TokensPrompt(prompt_token_ids=prompt_raw) + prompt = ( + TextPrompt(prompt=prompt_raw) + if isinstance(prompt_raw, str) + else TokensPrompt(prompt_token_ids=prompt_raw) + ) if mm_data_future is not None: prompt["multi_modal_data"] = await mm_data_future if mm_uuids is not None: diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index 1c39fc43d649..6536c7a43424 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -78,7 +78,7 @@ def __init__( safe_apply_chat_template, executor=self._apply_chat_template_executor ) - @property + @property # type: ignore[override] def tokenizer(self) -> MistralTokenizer | None: return self._tokenizer @@ -107,11 +107,12 @@ def render_messages( ) prompt_raw = safe_apply_chat_template(tokenizer, messages, **kwargs) - if isinstance(prompt_raw, str): - prompt = TextPrompt(prompt=prompt_raw) - else: - prompt = TokensPrompt(prompt_token_ids=prompt_raw) + prompt = ( + TextPrompt(prompt=prompt_raw) + if isinstance(prompt_raw, str) + else TokensPrompt(prompt_token_ids=prompt_raw) + ) if mm_data is not None: prompt["multi_modal_data"] = mm_data if mm_uuids is not None: @@ -134,11 +135,12 @@ async def render_messages_async( prompt_raw = await self._apply_chat_template_async( tokenizer, messages, **kwargs ) - if isinstance(prompt_raw, str): - prompt = TextPrompt(prompt=prompt_raw) - else: - prompt = TokensPrompt(prompt_token_ids=prompt_raw) + prompt = ( + TextPrompt(prompt=prompt_raw) + if isinstance(prompt_raw, str) + else TokensPrompt(prompt_token_ids=prompt_raw) + ) if mm_data_future is not None: prompt["multi_modal_data"] = await mm_data_future if mm_uuids is not None: diff --git a/vllm/renderers/terratorch.py b/vllm/renderers/terratorch.py index d1d7c842214c..5b05391170e1 100644 --- a/vllm/renderers/terratorch.py +++ b/vllm/renderers/terratorch.py @@ -30,7 +30,7 @@ def __init__(self, config: ModelConfig) -> None: if not config.skip_tokenizer_init: raise ValueError("Terratorch renderer requires `skip_tokenizer_init=True`") - @property + @property # type: ignore[override] def tokenizer(self) -> TokenizerLike | None: return None From 942c94d7e4a7fc35bfb13a28b77d772f0e4c1441 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:06:43 +0000 Subject: [PATCH 12/65] Allow `tokenizer=None` Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_chat.py | 13 +++++++++---- vllm/entrypoints/openai/serving_completion.py | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 1476aa8f38ce..591557dea392 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -189,7 +189,7 @@ async def create_chat_completion( model_name = self.models.model_name(lora_request) renderer = self.engine_client.renderer - tokenizer = renderer.get_tokenizer() + tokenizer = renderer.tokenizer tool_parser = self.tool_parser @@ -532,7 +532,7 @@ async def chat_completion_stream_generator( request_id: str, model_name: str, conversation: list[ConversationMessage], - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, request_metadata: RequestResponseMetadata, ) -> AsyncGenerator[str, None]: created_time = int(time.time()) @@ -1303,7 +1303,7 @@ async def chat_completion_full_generator( request_id: str, model_name: str, conversation: list[ConversationMessage], - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, request_metadata: RequestResponseMetadata, ) -> ErrorResponse | ChatCompletionResponse: created_time = int(time.time()) @@ -1655,7 +1655,7 @@ def _create_chat_logprobs( self, token_ids: GenericSequence[int], top_logprobs: GenericSequence[dict[int, Logprob] | None], - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, num_output_top_logprobs: int | None = None, return_as_token_id: bool | None = None, ) -> ChatCompletionLogProbs: @@ -1673,6 +1673,11 @@ def _create_chat_logprobs( if should_return_as_token_id: token = f"token_id:{token_id}" else: + if tokenizer is None: + raise ValueError( + "Unable to get tokenizer because `skip_tokenizer_init=True`" + ) + token = tokenizer.decode(token_id) logprobs_content.append( diff --git a/vllm/entrypoints/openai/serving_completion.py b/vllm/entrypoints/openai/serving_completion.py index d6b20b3f407f..caca2545d946 100644 --- a/vllm/entrypoints/openai/serving_completion.py +++ b/vllm/entrypoints/openai/serving_completion.py @@ -248,7 +248,7 @@ async def create_completion( stream = request.stream and not request.use_beam_search # Streaming response - tokenizer = self.renderer.get_tokenizer() + tokenizer = self.renderer.tokenizer if stream: return self.completion_stream_generator( From 86d1104a7c963c1bccc4a1c533eb911eca20dbe0 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:11:57 +0000 Subject: [PATCH 13/65] Handle tokenize Signed-off-by: DarkLight1337 --- vllm/tokenizers/deepseekv32.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/vllm/tokenizers/deepseekv32.py b/vllm/tokenizers/deepseekv32.py index 43dbb2b5c56c..cbca594bc0b6 100644 --- a/vllm/tokenizers/deepseekv32.py +++ b/vllm/tokenizers/deepseekv32.py @@ -45,7 +45,7 @@ def apply_chat_template( messages: list["ChatCompletionMessageParam"], tools: list[dict[str, Any]] | None = None, **kwargs, - ) -> str: + ) -> str | list[int]: thinking = kwargs.get("thinking", False) thinking_mode = "thinking" if not thinking: @@ -58,7 +58,19 @@ def apply_chat_template( messages[0]["tools"] = tools # type: ignore[typeddict-unknown-key] drop_thinking = False encode_config = dict(thinking_mode=thinking_mode, drop_thinking=drop_thinking) + prompt_str = encode_messages(messages, **encode_config) # type: ignore + + if kwargs.get("tokenize", True): + tokenizer_kwargs = { + k: kwargs[k] for k in ("truncation", "max_length") if k in kwargs + } + return self.encode( + prompt_str, + add_special_tokens=False, + **tokenizer_kwargs, + ) + return prompt_str def num_special_tokens_to_add(self) -> int: From 4be9517a943fb051aac7ff933db27ee09a0dbe26 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:13:29 +0000 Subject: [PATCH 14/65] Ignore mypy Signed-off-by: DarkLight1337 --- vllm/renderers/deepseekv32.py | 4 ++-- vllm/renderers/hf.py | 4 ++-- vllm/renderers/mistral.py | 4 ++-- vllm/tokenizers/protocol.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/vllm/renderers/deepseekv32.py b/vllm/renderers/deepseekv32.py index f1204515ebcb..13993244fca4 100644 --- a/vllm/renderers/deepseekv32.py +++ b/vllm/renderers/deepseekv32.py @@ -88,7 +88,7 @@ def render_messages( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt + return conversation, prompt # type: ignore[return-type] async def render_messages_async( self, @@ -118,4 +118,4 @@ async def render_messages_async( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt + return conversation, prompt # type: ignore[return-type] diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 9f39b0dc49dc..3d0d0ba57160 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -559,7 +559,7 @@ def render_messages( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt + return conversation, prompt # type: ignore[return-type] async def render_messages_async( self, @@ -599,4 +599,4 @@ async def render_messages_async( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt + return conversation, prompt # type: ignore[return-type] diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index 6536c7a43424..5d112d66d3d3 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -118,7 +118,7 @@ def render_messages( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt + return conversation, prompt # type: ignore[return-type] async def render_messages_async( self, @@ -146,4 +146,4 @@ async def render_messages_async( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt + return conversation, prompt # type: ignore[return-type] diff --git a/vllm/tokenizers/protocol.py b/vllm/tokenizers/protocol.py index d6a3b0ba9b5f..28754f9e10d0 100644 --- a/vllm/tokenizers/protocol.py +++ b/vllm/tokenizers/protocol.py @@ -97,7 +97,7 @@ def apply_chat_template( messages: list["ChatCompletionMessageParam"], tools: list[dict[str, Any]] | None = None, **kwargs, - ) -> list[int]: + ) -> str | list[int]: raise NotImplementedError def convert_tokens_to_string(self, tokens: list[str]) -> str: From bab3ff97818138f25b05b4a80e2d4ee890fec2cb Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:14:10 +0000 Subject: [PATCH 15/65] Fix Signed-off-by: DarkLight1337 --- vllm/renderers/terratorch.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/vllm/renderers/terratorch.py b/vllm/renderers/terratorch.py index 5b05391170e1..caeb7b9917dd 100644 --- a/vllm/renderers/terratorch.py +++ b/vllm/renderers/terratorch.py @@ -34,6 +34,11 @@ def __init__(self, config: ModelConfig) -> None: def tokenizer(self) -> TokenizerLike | None: return None + # NOTE: Remove this once LLM.tokenizer.setter is removed + @tokenizer.setter + def tokenizer(self, tokenizer: TokenizerLike | None) -> None: + self._tokenizer = tokenizer + def get_tokenizer(self) -> TokenizerLike: raise ValueError("Tokenizer not available for Terratorch renderer") From a40c5e44dafaaa804cac7e48966d882ca3f86ef4 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:33:28 +0000 Subject: [PATCH 16/65] Fix mypy Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_chat.py | 15 +++++++++++++++ vllm/entrypoints/openai/serving_engine.py | 12 ++++++++++-- vllm/renderers/deepseekv32.py | 4 ++-- vllm/renderers/hf.py | 6 +++--- vllm/renderers/mistral.py | 4 ++-- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 591557dea392..3ad50e329bed 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -586,6 +586,11 @@ async def chat_completion_stream_generator( try: if self.reasoning_parser: + if tokenizer is None: + raise ValueError( + "Tokenizer not available when `skip_tokenizer_init=True`" + ) + reasoning_parser = self.reasoning_parser( tokenizer, chat_template_kwargs=request.chat_template_kwargs, # type: ignore @@ -599,6 +604,11 @@ async def chat_completion_stream_generator( # Prepare the tool parser if it's needed try: if tool_choice_auto and self.tool_parser: + if tokenizer is None: + raise ValueError( + "Tokenizer not available when `skip_tokenizer_init=True`" + ) + tool_parsers: list[ToolParser | None] = [ self.tool_parser(tokenizer) ] * num_choices @@ -1350,6 +1360,11 @@ async def chat_completion_full_generator( reasoning = None if self.tool_parser is not None: + if tokenizer is None: + raise ValueError( + "Tokenizer not available when `skip_tokenizer_init=True`" + ) + tool_parser = self.tool_parser(tokenizer) # NOTE: We use token_ids for openai tool parser tool_call_info = tool_parser.extract_tool_calls( diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 2b21b2d47308..4710db70fa92 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -94,6 +94,7 @@ from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse from vllm.entrypoints.utils import _validate_truncation_size from vllm.inputs.data import PromptType, SingletonPrompt +from vllm.inputs.data import TextPrompt as EngineTextPrompt from vllm.inputs.data import TokensPrompt as EngineTokensPrompt from vllm.inputs.parse import ( PromptComponents, @@ -1092,6 +1093,7 @@ async def _preprocess_chat( **(chat_template_kwargs or {}), } + engine_prompt: EngineTokensPrompt | EngineTextPrompt conversation, engine_prompt = await renderer.render_messages_async( messages, chat_template_content_format=chat_template_content_format, @@ -1099,12 +1101,13 @@ async def _preprocess_chat( ) if "prompt_token_ids" not in engine_prompt: - engine_prompt = await self._tokenize_prompt_input_async( + tokenized_prompt = await self._tokenize_prompt_input_async( request, renderer.get_tokenizer(), engine_prompt["prompt"], add_special_tokens=add_special_tokens, ) + engine_prompt = EngineTokensPrompt(**tokenized_prompt) if request.mm_processor_kwargs is not None: engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs @@ -1344,7 +1347,7 @@ def _get_data_parallel_rank(raw_request: Request | None) -> int | None: @staticmethod def _parse_tool_calls_from_content( request: ResponsesRequest | ChatCompletionRequest, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, enable_auto_tools: bool, tool_parser_cls: Callable[[TokenizerLike], ToolParser] | None, content: str | None = None, @@ -1384,6 +1387,11 @@ def _parse_tool_calls_from_content( and enable_auto_tools and (request.tool_choice == "auto" or request.tool_choice is None) ): + if tokenizer is None: + raise ValueError( + "Tokenizer not available when `skip_tokenizer_init=True`" + ) + # Automatic Tool Call Parsing try: tool_parser = tool_parser_cls(tokenizer) diff --git a/vllm/renderers/deepseekv32.py b/vllm/renderers/deepseekv32.py index 13993244fca4..433c525f61cb 100644 --- a/vllm/renderers/deepseekv32.py +++ b/vllm/renderers/deepseekv32.py @@ -88,7 +88,7 @@ def render_messages( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt # type: ignore[return-type] + return conversation, prompt # type: ignore[return-value] async def render_messages_async( self, @@ -118,4 +118,4 @@ async def render_messages_async( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt # type: ignore[return-type] + return conversation, prompt # type: ignore[return-value] diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 3d0d0ba57160..939af5e3546b 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -524,7 +524,7 @@ def get_tokenizer(self) -> HfTokenizer: def render_messages( self, messages: list[ChatCompletionMessageParam], - chat_template_content_format: ChatTemplateContentFormat, + chat_template_content_format: ChatTemplateContentFormat = "auto", **kwargs, ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: model_config = self.config @@ -559,7 +559,7 @@ def render_messages( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt # type: ignore[return-type] + return conversation, prompt # type: ignore[return-value] async def render_messages_async( self, @@ -599,4 +599,4 @@ async def render_messages_async( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt # type: ignore[return-type] + return conversation, prompt # type: ignore[return-value] diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index 5d112d66d3d3..d65b51685813 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -118,7 +118,7 @@ def render_messages( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt # type: ignore[return-type] + return conversation, prompt # type: ignore[return-value] async def render_messages_async( self, @@ -146,4 +146,4 @@ async def render_messages_async( if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - return conversation, prompt # type: ignore[return-type] + return conversation, prompt # type: ignore[return-value] From 765285379a2bce31e9a7b6b7717f47bffd64bcdf Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:34:49 +0000 Subject: [PATCH 17/65] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_chat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 3ad50e329bed..e6f3ab78c0b4 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -1646,7 +1646,7 @@ def _get_top_logprobs( self, logprobs: dict[int, Logprob], top_logprobs: int | None, - tokenizer: TokenizerLike, + tokenizer: TokenizerLike | None, should_return_as_token_id: bool, ) -> list[ChatCompletionLogProb]: return [ From c709ef91d3af2590f88de11139be0228b8c6c7d8 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:37:20 +0000 Subject: [PATCH 18/65] Fix Signed-off-by: DarkLight1337 --- vllm/tokenizers/registry.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 4c50b5ddf24d..c5f1d471a3bd 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -3,7 +3,7 @@ import importlib.util from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, Literal, TypeVar +from typing import TYPE_CHECKING, TypeVar import huggingface_hub from typing_extensions import assert_never, deprecated @@ -23,7 +23,7 @@ from .protocol import TokenizerLike if TYPE_CHECKING: - from vllm.config import ModelConfig + from vllm.config.model import ModelConfig, RunnerType logger = init_logger(__name__) @@ -78,7 +78,7 @@ def load_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: def resolve_tokenizer_args( tokenizer_name: str | Path, *args, - runner_type: Literal["generate", "pooling"] = "generate", + runner_type: "RunnerType" = "generate", tokenizer_mode: str = "auto", trust_remote_code: bool = False, revision: str | None = None, From 7b9aa93c98b0a0652a9d967a6a83c2fda3957f7b Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 10:46:32 +0000 Subject: [PATCH 19/65] mypy Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_chat.py | 5 +++++ vllm/entrypoints/openai/serving_engine.py | 2 +- vllm/renderers/hf.py | 2 +- vllm/renderers/terratorch.py | 7 ++++++- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index e6f3ab78c0b4..117a6ae1072f 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -1407,6 +1407,11 @@ async def chat_completion_full_generator( if self.reasoning_parser: try: + if tokenizer is None: + raise ValueError( + "Tokenizer not available when `skip_tokenizer_init=True`" + ) + reasoning_parser = self.reasoning_parser( tokenizer, chat_template_kwargs=request.chat_template_kwargs, # type: ignore diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 4710db70fa92..e6382a454cf1 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1132,7 +1132,7 @@ async def _preprocess_chat( tokenizer = renderer.get_tokenizer() request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore - return conversation, [engine_prompt], [engine_prompt] + return conversation, [engine_prompt], [engine_prompt] # type: ignore[list-item] async def _process_inputs( self, diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 939af5e3546b..fda5d1248a1a 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -524,7 +524,7 @@ def get_tokenizer(self) -> HfTokenizer: def render_messages( self, messages: list[ChatCompletionMessageParam], - chat_template_content_format: ChatTemplateContentFormat = "auto", + chat_template_content_format: ChatTemplateContentFormatOption = "auto", **kwargs, ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: model_config = self.config diff --git a/vllm/renderers/terratorch.py b/vllm/renderers/terratorch.py index caeb7b9917dd..34248222a967 100644 --- a/vllm/renderers/terratorch.py +++ b/vllm/renderers/terratorch.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( @@ -19,7 +20,11 @@ class TerratorchRenderer(RendererLike): @classmethod - def from_config(cls, config: ModelConfig) -> "RendererLike": + def from_config( + cls, + config: "ModelConfig", + tokenizer_kwargs: dict[str, Any], + ) -> "RendererLike": return cls(config) def __init__(self, config: ModelConfig) -> None: From 45558418c36f3709a7e3780586971216507f9e45 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 12:26:33 +0000 Subject: [PATCH 20/65] Fix tests Signed-off-by: DarkLight1337 --- tests/reasoning/test_mistral_reasoning_parser.py | 2 +- tests/reasoning/utils.py | 2 +- tests/tokenizers_/test_detokenize.py | 2 +- tests/tool_use/test_mistral_tool_parser.py | 7 ++----- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/reasoning/test_mistral_reasoning_parser.py b/tests/reasoning/test_mistral_reasoning_parser.py index 0fe315c2567f..36db65f99cef 100644 --- a/tests/reasoning/test_mistral_reasoning_parser.py +++ b/tests/reasoning/test_mistral_reasoning_parser.py @@ -5,7 +5,7 @@ from tests.reasoning.utils import run_reasoning_extraction_mistral from vllm.reasoning import ReasoningParser, ReasoningParserManager -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer parser_name = "mistral" diff --git a/tests/reasoning/utils.py b/tests/reasoning/utils.py index 695312a0cadf..a020fb8e9716 100644 --- a/tests/reasoning/utils.py +++ b/tests/reasoning/utils.py @@ -4,7 +4,7 @@ from vllm.entrypoints.openai.protocol import ChatCompletionRequest, DeltaMessage from vllm.reasoning import ReasoningParser -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer class StreamingReasoningReconstructor: diff --git a/tests/tokenizers_/test_detokenize.py b/tests/tokenizers_/test_detokenize.py index ae1d6b095672..d307993d04df 100644 --- a/tests/tokenizers_/test_detokenize.py +++ b/tests/tokenizers_/test_detokenize.py @@ -8,7 +8,7 @@ from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from vllm.sampling_params import SamplingParams -from vllm.tokenizers import MistralTokenizer +from vllm.tokenizers.mistral import MistralTokenizer from vllm.v1.engine import EngineCoreRequest from vllm.v1.engine.detokenizer import ( FastIncrementalDetokenizer, diff --git a/tests/tool_use/test_mistral_tool_parser.py b/tests/tool_use/test_mistral_tool_parser.py index e5deb7f40eb3..f128046c65f1 100644 --- a/tests/tool_use/test_mistral_tool_parser.py +++ b/tests/tool_use/test_mistral_tool_parser.py @@ -13,12 +13,9 @@ from vllm.entrypoints.openai.protocol import DeltaMessage, DeltaToolCall from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolParser -from vllm.tokenizers import ( - MistralTokenizer, - TokenizerLike, - get_tokenizer, -) +from vllm.tokenizers import TokenizerLike, get_tokenizer from vllm.tokenizers.detokenizer_utils import detokenize_incrementally +from vllm.tokenizers.mistral import MistralTokenizer @pytest.fixture(scope="module") From 8ebde239d881bb7331c26b011dce5283e19ce614 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 12:40:21 +0000 Subject: [PATCH 21/65] Simplify Signed-off-by: DarkLight1337 --- vllm/renderers/deepseekv32.py | 7 +++-- vllm/renderers/hf.py | 6 ++--- vllm/renderers/mistral.py | 7 +++-- vllm/tokenizers/__init__.py | 4 --- vllm/tokenizers/registry.py | 51 +++++++++-------------------------- 5 files changed, 26 insertions(+), 49 deletions(-) diff --git a/vllm/renderers/deepseekv32.py b/vllm/renderers/deepseekv32.py index 433c525f61cb..eeac6add25a4 100644 --- a/vllm/renderers/deepseekv32.py +++ b/vllm/renderers/deepseekv32.py @@ -11,7 +11,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import cached_init_tokenizer +from vllm.tokenizers import cached_get_tokenizer from vllm.tokenizers.deepseekv32 import DeepseekV32Tokenizer from .protocol import RendererLike @@ -40,7 +40,10 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = cached_init_tokenizer(DeepseekV32Tokenizer, **tokenizer_kwargs) + tokenizer = cached_get_tokenizer( + tokenizer_cls=DeepseekV32Tokenizer, + **tokenizer_kwargs, + ) self._tokenizer = tokenizer diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index fda5d1248a1a..cf6556a7f57c 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -25,7 +25,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import cached_init_tokenizer +from vllm.tokenizers import cached_get_tokenizer from vllm.tokenizers.hf import CachedHfTokenizer, HfTokenizer from vllm.transformers_utils.chat_templates import get_chat_template_fallback_path from vllm.transformers_utils.processor import cached_get_processor @@ -497,8 +497,8 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = cached_init_tokenizer( - CachedHfTokenizer, # type: ignore[type-abstract] + tokenizer = cached_get_tokenizer( + tokenizer_cls=CachedHfTokenizer, # type: ignore[type-abstract] **tokenizer_kwargs, ) assert isinstance(tokenizer, HfTokenizer) diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index d65b51685813..2d02e88bee14 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -12,7 +12,7 @@ ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger -from vllm.tokenizers import cached_init_tokenizer +from vllm.tokenizers import cached_get_tokenizer from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.async_utils import make_async @@ -69,7 +69,10 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = cached_init_tokenizer(MistralTokenizer, **tokenizer_kwargs) + tokenizer = cached_get_tokenizer( + tokenizer_cls=MistralTokenizer, + **tokenizer_kwargs, + ) self._tokenizer = tokenizer diff --git a/vllm/tokenizers/__init__.py b/vllm/tokenizers/__init__.py index a0afc174be9c..31e74b1a16e2 100644 --- a/vllm/tokenizers/__init__.py +++ b/vllm/tokenizers/__init__.py @@ -5,10 +5,8 @@ from .registry import ( TokenizerRegistry, cached_get_tokenizer, - cached_init_tokenizer, cached_tokenizer_from_config, get_tokenizer, - init_tokenizer, init_tokenizer_from_config, ) @@ -18,7 +16,5 @@ "cached_get_tokenizer", "get_tokenizer", "cached_tokenizer_from_config", - "cached_init_tokenizer", - "init_tokenizer", "init_tokenizer_from_config", ] diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index c5f1d471a3bd..75f3381bc6f5 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -3,10 +3,10 @@ import importlib.util from functools import lru_cache from pathlib import Path -from typing import TYPE_CHECKING, TypeVar +from typing import TYPE_CHECKING import huggingface_hub -from typing_extensions import assert_never, deprecated +from typing_extensions import TypeVar, assert_never, deprecated import vllm.envs as envs from vllm.logger import init_logger @@ -178,14 +178,18 @@ def tokenizer_args_from_config(config: "ModelConfig", **kwargs): ) +_T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike) + + def get_tokenizer( tokenizer_name: str | Path, *args, + tokenizer_cls: type[_T] = TokenizerLike, trust_remote_code: bool = False, revision: str | None = None, download_dir: str | None = None, **kwargs, -) -> TokenizerLike: +) -> _T: """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" tokenizer_mode, tokenizer_args, tokenizer_kwargs = cached_resolve_tokenizer_args( tokenizer_name, @@ -197,8 +201,12 @@ def get_tokenizer( ) tokenizer_kwargs.update(kwargs) - tokenizer_cls = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode) - tokenizer = tokenizer_cls.from_pretrained(*tokenizer_args, **tokenizer_kwargs) + if tokenizer_cls == TokenizerLike: + tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode) + else: + tokenizer_cls_ = tokenizer_cls + + tokenizer = tokenizer_cls_.from_pretrained(*tokenizer_args, **tokenizer_kwargs) if not tokenizer.is_fast: logger.warning( "Using a slow tokenizer. This might cause a significant " @@ -230,36 +238,3 @@ def cached_tokenizer_from_config(model_config: "ModelConfig", **kwargs): ) def init_tokenizer_from_config(model_config: "ModelConfig"): return cached_tokenizer_from_config(model_config) - - -_T = TypeVar("_T", bound=TokenizerLike) - - -def init_tokenizer( - tokenizer_cls: type[_T], - tokenizer_name: str | Path, - trust_remote_code: bool = False, - revision: str | None = None, - download_dir: str | None = None, - **kwargs, -) -> _T: - tokenizer_mode, tokenizer_args, tokenizer_kwargs = cached_resolve_tokenizer_args( - tokenizer_name, - trust_remote_code=trust_remote_code, - revision=revision, - download_dir=download_dir, - **kwargs, - ) - tokenizer_kwargs.update(kwargs) - - tokenizer = tokenizer_cls.from_pretrained(*tokenizer_args, **tokenizer_kwargs) - if not tokenizer.is_fast: - logger.warning( - "Using a slow tokenizer. This might cause a significant " - "slowdown. Consider using a fast tokenizer instead." - ) - - return tokenizer # type: ignore - - -cached_init_tokenizer = lru_cache(init_tokenizer) From 2ff75320059a808242c7f7633e2513722430561d Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 12:46:53 +0000 Subject: [PATCH 22/65] Fix mypy Signed-off-by: DarkLight1337 --- vllm/tokenizers/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 75f3381bc6f5..82e3f2c41630 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -184,7 +184,7 @@ def tokenizer_args_from_config(config: "ModelConfig", **kwargs): def get_tokenizer( tokenizer_name: str | Path, *args, - tokenizer_cls: type[_T] = TokenizerLike, + tokenizer_cls: type[_T] = TokenizerLike, # type: ignore[assignment] trust_remote_code: bool = False, revision: str | None = None, download_dir: str | None = None, From 2f2aaf781238bc86a0d7a873a357ed4aae725cc3 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 12:48:39 +0000 Subject: [PATCH 23/65] Update imports Signed-off-by: DarkLight1337 --- vllm/transformers_utils/tokenizer.py | 6 +++--- vllm/v1/structured_output/__init__.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/vllm/transformers_utils/tokenizer.py b/vllm/transformers_utils/tokenizer.py index 32999903b348..94aeecbd3543 100644 --- a/vllm/transformers_utils/tokenizer.py +++ b/vllm/transformers_utils/tokenizer.py @@ -60,17 +60,17 @@ def __getattr__(name: str): return cached_tokenizer_from_config if name == "init_tokenizer_from_configs": - from vllm.tokenizers import init_tokenizer_from_config + from vllm.tokenizers import cached_tokenizer_from_config warnings.warn( "`vllm.transformers_utils.tokenizer.init_tokenizer_from_configs` " - "has been moved to `vllm.tokenizers.init_tokenizer_from_config`. " + "has been moved to `vllm.tokenizers.cached_tokenizer_from_config`. " "The old name will be removed in v0.13.", DeprecationWarning, stacklevel=2, ) - return init_tokenizer_from_config + return cached_tokenizer_from_config raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/vllm/v1/structured_output/__init__.py b/vllm/v1/structured_output/__init__.py index 5ee88178cdf6..c0c25ca92911 100644 --- a/vllm/v1/structured_output/__init__.py +++ b/vllm/v1/structured_output/__init__.py @@ -7,7 +7,7 @@ from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.reasoning import ReasoningParserManager -from vllm.tokenizers import init_tokenizer_from_config +from vllm.tokenizers import cached_tokenizer_from_config from vllm.utils.import_utils import LazyLoader from vllm.v1.structured_output.backend_guidance import GuidanceBackend from vllm.v1.structured_output.backend_types import ( @@ -71,7 +71,7 @@ def __init__(self, vllm_config: VllmConfig): # of CPUs. max_workers = max(1, (multiprocessing.cpu_count() + 1) // 2) self.executor = ThreadPoolExecutor(max_workers=max_workers) - self.tokenizer = init_tokenizer_from_config( + self.tokenizer = cached_tokenizer_from_config( model_config=self.vllm_config.model_config ) reasoning_parser = ( From 05e9b469ae769074e9789b7176a18c3817a419db Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 12:51:52 +0000 Subject: [PATCH 24/65] Fix Signed-off-by: DarkLight1337 --- tests/entrypoints/openai/test_chat_template.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/entrypoints/openai/test_chat_template.py b/tests/entrypoints/openai/test_chat_template.py index 4db7c9576b1d..7d912af39424 100644 --- a/tests/entrypoints/openai/test_chat_template.py +++ b/tests/entrypoints/openai/test_chat_template.py @@ -8,7 +8,7 @@ from vllm.entrypoints.openai.protocol import ChatCompletionRequest from vllm.renderers.hf import safe_apply_chat_template from vllm.tokenizers import get_tokenizer -from vllm.tokenizers.hf import HfTokenizer +from vllm.tokenizers.hf import CachedHfTokenizer from ...models.registry import HF_EXAMPLE_MODELS from ...utils import VLLM_PATH @@ -125,8 +125,8 @@ def test_get_gen_prompt( # Initialize the tokenizer tokenizer = get_tokenizer( - HfTokenizer, - tokenizer_name=model_config.tokenizer, + model_config.tokenizer, + tokenizer_cls=CachedHfTokenizer, trust_remote_code=model_config.trust_remote_code, ) template_content = load_chat_template(chat_template=template) From 10460d64ce0d6f3c107622cfef7b99162152145f Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 12:58:11 +0000 Subject: [PATCH 25/65] Fix Signed-off-by: DarkLight1337 --- vllm/renderers/registry.py | 1 - vllm/tokenizers/registry.py | 1 - 2 files changed, 2 deletions(-) diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 39e2b54d70e3..088f0b72206a 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -71,7 +71,6 @@ def renderer_from_config(config: "ModelConfig", **kwargs): tokenizer_mode, tokenizer_args, tokenizer_kwargs = tokenizer_args_from_config( config, **kwargs ) - tokenizer_kwargs.update(kwargs) tokenizer_kwargs["tokenizer_name"] = tokenizer_args[0] if config.tokenizer_mode == "auto" and config.model_impl == "terratorch": diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 82e3f2c41630..8fe67a7b70ce 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -199,7 +199,6 @@ def get_tokenizer( download_dir=download_dir, **kwargs, ) - tokenizer_kwargs.update(kwargs) if tokenizer_cls == TokenizerLike: tokenizer_cls_ = TokenizerRegistry.load_tokenizer_cls(tokenizer_mode) From 49d10ff9c06c8970fcca4b065da2b536c57c9d82 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 13:05:45 +0000 Subject: [PATCH 26/65] Simplify Signed-off-by: DarkLight1337 --- vllm/renderers/registry.py | 37 ++++++++++++++++++++----------------- vllm/tokenizers/registry.py | 33 +++++++++++++++++---------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 088f0b72206a..e94ade060ecf 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -1,5 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any from vllm.logger import init_logger @@ -14,15 +15,21 @@ logger = init_logger(__name__) -class RendererRegistry: - def __init__(self) -> None: - super().__init__() +_VLLM_RENDERERS = { + "deepseekv32": ("deepseekv32", "DeepseekV32Renderer"), + "hf": ("hf", "HfRenderer"), + "mistral": ("mistral", "MistralRenderer"), + "terratorch": ("terratorch", "TerratorchRenderer"), +} + - # Renderer name -> (renderer module, renderer class) - self._registry: dict[str, tuple[str, str]] = {} +@dataclass +class RendererRegistry: + # Renderer name -> (renderer module, renderer class) + renderers: dict[str, tuple[str, str]] = field(default_factory=dict) def register(self, renderer_mode: str, module: str, class_name: str) -> None: - if renderer_mode in self._registry: + if renderer_mode in self.renderers: logger.warning( "%s.%s is already registered for renderer_mode=%r. " "It is overwritten by the new one.", @@ -31,15 +38,15 @@ def register(self, renderer_mode: str, module: str, class_name: str) -> None: renderer_mode, ) - self._registry[renderer_mode] = (module, class_name) + self.renderers[renderer_mode] = (module, class_name) return None def load_renderer_cls(self, renderer_mode: str) -> type[RendererLike]: - if renderer_mode not in self._registry: + if renderer_mode not in self.renderers: raise ValueError(f"No renderer registered for {renderer_mode=!r}.") - module, class_name = self._registry[renderer_mode] + module, class_name = self.renderers[renderer_mode] logger.debug_once(f"Loading {class_name} for {renderer_mode=!r}") return resolve_obj_by_qualname(f"{module}.{class_name}") @@ -54,17 +61,13 @@ def load_renderer( return renderer_cls.from_config(config, tokenizer_kwargs) -RENDERER_REGISTRY = RendererRegistry() -"""The global `RendererRegistry` instance.""" - -RENDERER_REGISTRY._registry.update( +RENDERER_REGISTRY = RendererRegistry( { - "deepseekv32": ("vllm.renderers.deepseekv32", "DeepseekV32Renderer"), - "hf": ("vllm.renderers.hf", "HfRenderer"), - "mistral": ("vllm.renderers.mistral", "MistralRenderer"), - "terratorch": ("vllm.renderers.terratorch", "TerratorchRenderer"), + key: (f"vllm.renderers.{mod_relname}", cls_name) + for key, (mod_relname, cls_name) in _VLLM_RENDERERS.items() } ) +"""The global `RendererRegistry` instance.""" def renderer_from_config(config: "ModelConfig", **kwargs): diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 8fe67a7b70ce..0e04e92180d7 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import importlib.util +from dataclasses import dataclass, field from functools import lru_cache from pathlib import Path from typing import TYPE_CHECKING @@ -28,15 +29,19 @@ logger = init_logger(__name__) -class TokenizerRegistryType: - def __init__(self) -> None: - super().__init__() +_VLLM_TOKENIZERS = { + "deepseekv32": ("deepseekv32", "DeepseekV32Tokenizer"), + "hf": ("hf", "CachedHfTokenizer"), + "mistral": ("mistral", "MistralTokenizer"), +} - # Tokenizer name -> (tokenizer module, tokenizer class) - self._registry: dict[str, tuple[str, str]] = {} + +@dataclass +class _TokenizerRegistry: + tokenizers: dict[str, tuple[str, str]] = field(default_factory=dict) def register(self, tokenizer_mode: str, module: str, class_name: str) -> None: - if tokenizer_mode in self._registry: + if tokenizer_mode in self.tokenizers: logger.warning( "%s.%s is already registered for tokenizer_mode=%r. " "It is overwritten by the new one.", @@ -45,15 +50,15 @@ def register(self, tokenizer_mode: str, module: str, class_name: str) -> None: tokenizer_mode, ) - self._registry[tokenizer_mode] = (module, class_name) + self.tokenizers[tokenizer_mode] = (module, class_name) return None def load_tokenizer_cls(self, tokenizer_mode: str) -> type[TokenizerLike]: - if tokenizer_mode not in self._registry: + if tokenizer_mode not in self.tokenizers: raise ValueError(f"No tokenizer registered for {tokenizer_mode=!r}.") - module, class_name = self._registry[tokenizer_mode] + module, class_name = self.tokenizers[tokenizer_mode] logger.debug_once(f"Loading {class_name} for {tokenizer_mode=!r}") return resolve_obj_by_qualname(f"{module}.{class_name}") @@ -63,14 +68,10 @@ def load_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: return tokenizer_cls.from_pretrained(*args, **kwargs) -TokenizerRegistry = TokenizerRegistryType() -"""The global `TokenizerRegistryType` instance.""" - -TokenizerRegistry._registry.update( +TokenizerRegistry = _TokenizerRegistry( { - "deepseekv32": ("vllm.tokenizers.deepseekv32", "DeepseekV32Tokenizer"), - "hf": ("vllm.tokenizers.hf", "CachedHfTokenizer"), - "mistral": ("vllm.tokenizers.mistral", "MistralTokenizer"), + key: (f"vllm.tokenizers.{mod_relname}", cls_name) + for key, (mod_relname, cls_name) in _VLLM_TOKENIZERS.items() } ) From 41d5ffa07864acc27cdc1f00fca30ab1ef4eed1e Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 13:07:13 +0000 Subject: [PATCH 27/65] Rename Signed-off-by: DarkLight1337 --- vllm/renderers/registry.py | 6 +++--- vllm/tokenizers/registry.py | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index e94ade060ecf..eb0692ab8946 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -25,7 +25,7 @@ @dataclass class RendererRegistry: - # Renderer name -> (renderer module, renderer class) + # Renderer mode -> (renderer module, renderer class) renderers: dict[str, tuple[str, str]] = field(default_factory=dict) def register(self, renderer_mode: str, module: str, class_name: str) -> None: @@ -63,8 +63,8 @@ def load_renderer( RENDERER_REGISTRY = RendererRegistry( { - key: (f"vllm.renderers.{mod_relname}", cls_name) - for key, (mod_relname, cls_name) in _VLLM_RENDERERS.items() + mode: (f"vllm.renderers.{mod_relname}", cls_name) + for mode, (mod_relname, cls_name) in _VLLM_RENDERERS.items() } ) """The global `RendererRegistry` instance.""" diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 0e04e92180d7..05a7e51abf08 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -38,6 +38,7 @@ @dataclass class _TokenizerRegistry: + # Tokenizer mode -> (tokenizer module, tokenizer class) tokenizers: dict[str, tuple[str, str]] = field(default_factory=dict) def register(self, tokenizer_mode: str, module: str, class_name: str) -> None: @@ -70,8 +71,8 @@ def load_tokenizer(self, tokenizer_mode: str, *args, **kwargs) -> TokenizerLike: TokenizerRegistry = _TokenizerRegistry( { - key: (f"vllm.tokenizers.{mod_relname}", cls_name) - for key, (mod_relname, cls_name) in _VLLM_TOKENIZERS.items() + mode: (f"vllm.tokenizers.{mod_relname}", cls_name) + for mode, (mod_relname, cls_name) in _VLLM_TOKENIZERS.items() } ) From 06ff81136171287be01a0c18b5a37824a1beedff Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 13:13:39 +0000 Subject: [PATCH 28/65] Fix Signed-off-by: DarkLight1337 --- tests/v1/engine/test_process_multi_modal_uuids.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/v1/engine/test_process_multi_modal_uuids.py b/tests/v1/engine/test_process_multi_modal_uuids.py index 46a79790a212..16c2f1b8ccd3 100644 --- a/tests/v1/engine/test_process_multi_modal_uuids.py +++ b/tests/v1/engine/test_process_multi_modal_uuids.py @@ -12,7 +12,6 @@ MultiModalConfig, VllmConfig, ) -from vllm.renderers.terratorch import TerratorchRenderer from vllm.sampling_params import SamplingParams from vllm.v1.engine import input_processor as input_processor_mod from vllm.v1.engine.input_processor import InputProcessor @@ -63,7 +62,7 @@ def _mock_input_processor( device_config=DeviceConfig(device="cpu"), ) - return InputProcessor(vllm_config, renderer=TerratorchRenderer(model_config, {})) + return InputProcessor(vllm_config) def test_multi_modal_uuids_length_mismatch_raises(monkeypatch): From cd8c8fb702b69b9f6952eb226df8cdf325e220c5 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 7 Dec 2025 13:18:52 +0000 Subject: [PATCH 29/65] Fix test Signed-off-by: DarkLight1337 --- tests/v1/engine/test_process_multi_modal_uuids.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/v1/engine/test_process_multi_modal_uuids.py b/tests/v1/engine/test_process_multi_modal_uuids.py index 16c2f1b8ccd3..c5158998f3b2 100644 --- a/tests/v1/engine/test_process_multi_modal_uuids.py +++ b/tests/v1/engine/test_process_multi_modal_uuids.py @@ -50,14 +50,17 @@ def _mock_input_processor( monkeypatch.setattr(VllmConfig, "__post_init__", lambda self: None, raising=True) model_config = ModelConfig( + tokenizer="dummy", skip_tokenizer_init=True, max_model_len=128, mm_processor_cache_gb=mm_cache_gb, generation_config="vllm", ) + model_config.runner_type = "generate" model_config.multimodal_config = MultiModalConfig(mm_processor_cache_gb=mm_cache_gb) vllm_config = VllmConfig( + model_config=model_config, cache_config=CacheConfig(enable_prefix_caching=enable_prefix_caching), device_config=DeviceConfig(device="cpu"), ) From fe2fbc919485fc8b73c39d6252f8288ea7e7d273 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 04:23:40 +0000 Subject: [PATCH 30/65] Fix wrong truncation side Signed-off-by: DarkLight1337 --- tests/tokenizers_/test_registry.py | 32 +++++++++++++++++++++++++- vllm/renderers/registry.py | 6 ++--- vllm/tokenizers/registry.py | 37 +++++++++++++++++------------- 3 files changed, 55 insertions(+), 20 deletions(-) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index 3a2f84ea976b..9ab2cd7305f9 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -2,7 +2,14 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from pathlib import Path -from vllm.tokenizers import TokenizerLike, TokenizerRegistry, get_tokenizer +import pytest + +from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.registry import ( + TokenizerRegistry, + get_tokenizer, + resolve_tokenizer_args, +) class TestTokenizer(TokenizerLike): @@ -40,6 +47,29 @@ def is_fast(self) -> bool: return True +@pytest.mark.parametrize("runner_type", ["generate", "pooling"]) +def test_resolve_tokenizer_args_idempotent(runner_type): + tokenizer_name = "facebook/opt-125m" + + tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs = ( + resolve_tokenizer_args( + tokenizer_name, + runner_type=runner_type, + ) + ) + + assert ( + tokenizer_mode, + tokenizer_name, + tokenizer_args, + tokenizer_kwargs, + ) == resolve_tokenizer_args( + tokenizer_name, + *tokenizer_args, + **tokenizer_kwargs, + ) + + def test_customized_tokenizer(): TokenizerRegistry.register("test_tokenizer", __name__, TestTokenizer.__name__) diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index eb0692ab8946..39aed8549a25 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -71,10 +71,10 @@ def load_renderer( def renderer_from_config(config: "ModelConfig", **kwargs): - tokenizer_mode, tokenizer_args, tokenizer_kwargs = tokenizer_args_from_config( - config, **kwargs + tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs = ( + tokenizer_args_from_config(config, **kwargs) ) - tokenizer_kwargs["tokenizer_name"] = tokenizer_args[0] + tokenizer_kwargs["tokenizer_name"] = tokenizer_name if config.tokenizer_mode == "auto" and config.model_impl == "terratorch": renderer_mode = "terratorch" diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 05a7e51abf08..7d0b45aed161 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -127,12 +127,13 @@ def resolve_tokenizer_args( ) tokenizer_kwargs["gguf_file"] = gguf_file - if runner_type == "generate" or runner_type == "draft": - tokenizer_kwargs["truncation_side"] = "left" - elif runner_type == "pooling": - tokenizer_kwargs["truncation_side"] = "right" - else: - assert_never(runner_type) + if "truncation_side" not in tokenizer_kwargs: + if runner_type == "generate" or runner_type == "draft": + tokenizer_kwargs["truncation_side"] = "left" + elif runner_type == "pooling": + tokenizer_kwargs["truncation_side"] = "right" + else: + assert_never(runner_type) if tokenizer_mode == "slow": if tokenizer_kwargs.get("use_fast", False): @@ -156,14 +157,14 @@ def resolve_tokenizer_args( if tokenizer_mode == "auto": tokenizer_mode = "hf" - tokenizer_args = (tokenizer_name, *args) + tokenizer_args = args tokenizer_kwargs = { "trust_remote_code": trust_remote_code, "revision": revision, **tokenizer_kwargs, } - return tokenizer_mode, tokenizer_args, tokenizer_kwargs + return tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args) @@ -193,13 +194,15 @@ def get_tokenizer( **kwargs, ) -> _T: """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" - tokenizer_mode, tokenizer_args, tokenizer_kwargs = cached_resolve_tokenizer_args( - tokenizer_name, - *args, - trust_remote_code=trust_remote_code, - revision=revision, - download_dir=download_dir, - **kwargs, + tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs = ( + cached_resolve_tokenizer_args( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + download_dir=download_dir, + **kwargs, + ) ) if tokenizer_cls == TokenizerLike: @@ -207,7 +210,9 @@ def get_tokenizer( else: tokenizer_cls_ = tokenizer_cls - tokenizer = tokenizer_cls_.from_pretrained(*tokenizer_args, **tokenizer_kwargs) + tokenizer = tokenizer_cls_.from_pretrained( + tokenizer_name, *tokenizer_args, **tokenizer_kwargs + ) if not tokenizer.is_fast: logger.warning( "Using a slow tokenizer. This might cause a significant " From f7c1b2b83694d390578218ce49423194e074612b Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:09:04 +0000 Subject: [PATCH 31/65] Update Signed-off-by: DarkLight1337 --- tests/tokenizers_/test_basic.py | 47 ++++++++++++++-------------- tests/tokenizers_/test_registry.py | 19 +++--------- vllm/renderers/hf.py | 12 +++++--- vllm/renderers/registry.py | 11 ++++--- vllm/tokenizers/registry.py | 49 +++++++++++------------------- 5 files changed, 61 insertions(+), 77 deletions(-) diff --git a/tests/tokenizers_/test_basic.py b/tests/tokenizers_/test_basic.py index b152227a5a50..0510261eacde 100644 --- a/tests/tokenizers_/test_basic.py +++ b/tests/tokenizers_/test_basic.py @@ -3,38 +3,39 @@ from typing import _get_protocol_attrs # type: ignore import pytest -from transformers import PreTrainedTokenizerBase +from transformers import ( + PreTrainedTokenizer, + PreTrainedTokenizerBase, + PreTrainedTokenizerFast, +) from vllm.tokenizers import TokenizerLike, get_tokenizer +from vllm.tokenizers.mistral import MistralTokenizer def _get_missing_attrs(obj: object, target: type): return [k for k in _get_protocol_attrs(target) if not hasattr(obj, k)] +def _assert_tokenizer_like(tokenizer: object): + missing_attrs = _get_missing_attrs(tokenizer, TokenizerLike) + assert not missing_attrs, f"Missing attrs: {missing_attrs}" + + def test_tokenizer_like_protocol(): - assert not ( - missing_attrs := _get_missing_attrs( - get_tokenizer("gpt2", use_fast=False), - TokenizerLike, - ) - ), f"Missing attrs: {missing_attrs}" - - assert not ( - missing_attrs := _get_missing_attrs( - get_tokenizer("gpt2", use_fast=True), - TokenizerLike, - ) - ), f"Missing attrs: {missing_attrs}" - - assert not ( - missing_attrs := _get_missing_attrs( - get_tokenizer( - "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral" - ), - TokenizerLike, - ) - ), f"Missing attrs: {missing_attrs}" + tokenizer = get_tokenizer("gpt2", use_fast=False) + assert isinstance(tokenizer, PreTrainedTokenizer) + _assert_tokenizer_like(tokenizer) + + tokenizer = get_tokenizer("gpt2", use_fast=True) + assert isinstance(tokenizer, PreTrainedTokenizerFast) + _assert_tokenizer_like(tokenizer) + + tokenizer = get_tokenizer( + "mistralai/Mistral-7B-Instruct-v0.3", tokenizer_mode="mistral" + ) + assert isinstance(tokenizer, MistralTokenizer) + _assert_tokenizer_like(tokenizer) @pytest.mark.parametrize("tokenizer_name", ["facebook/opt-125m", "gpt2"]) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index 9ab2cd7305f9..0c718edffff4 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -51,22 +51,13 @@ def is_fast(self) -> bool: def test_resolve_tokenizer_args_idempotent(runner_type): tokenizer_name = "facebook/opt-125m" - tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs = ( - resolve_tokenizer_args( - tokenizer_name, - runner_type=runner_type, - ) + tokenizer_mode, tokenizer_name, args, kwargs = resolve_tokenizer_args( + tokenizer_name, + runner_type=runner_type, ) - assert ( - tokenizer_mode, - tokenizer_name, - tokenizer_args, - tokenizer_kwargs, - ) == resolve_tokenizer_args( - tokenizer_name, - *tokenizer_args, - **tokenizer_kwargs, + assert (tokenizer_mode, tokenizer_name, args, kwargs) == resolve_tokenizer_args( + tokenizer_name, *args, **kwargs ) diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index cf6556a7f57c..a948a406e977 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -4,7 +4,7 @@ from collections import deque from collections.abc import Set from functools import lru_cache -from typing import Any +from typing import Any, cast import jinja2 import jinja2.ext @@ -497,11 +497,13 @@ def __init__( if config.skip_tokenizer_init: tokenizer = None else: - tokenizer = cached_get_tokenizer( - tokenizer_cls=CachedHfTokenizer, # type: ignore[type-abstract] - **tokenizer_kwargs, + tokenizer = cast( + HfTokenizer, + cached_get_tokenizer( + tokenizer_cls=CachedHfTokenizer, # type: ignore[type-abstract] + **tokenizer_kwargs, + ), ) - assert isinstance(tokenizer, HfTokenizer) self._tokenizer = tokenizer diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 39aed8549a25..a6c402edf32a 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -71,14 +71,17 @@ def load_renderer( def renderer_from_config(config: "ModelConfig", **kwargs): - tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs = ( - tokenizer_args_from_config(config, **kwargs) + tokenizer_mode, tokenizer_name, args, kwargs = tokenizer_args_from_config( + config, **kwargs ) - tokenizer_kwargs["tokenizer_name"] = tokenizer_name if config.tokenizer_mode == "auto" and config.model_impl == "terratorch": renderer_mode = "terratorch" else: renderer_mode = tokenizer_mode - return RENDERER_REGISTRY.load_renderer(renderer_mode, config, tokenizer_kwargs) + return RENDERER_REGISTRY.load_renderer( + renderer_mode, + config, + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name}, + ) diff --git a/vllm/tokenizers/registry.py b/vllm/tokenizers/registry.py index 7d0b45aed161..1296ce62ae69 100644 --- a/vllm/tokenizers/registry.py +++ b/vllm/tokenizers/registry.py @@ -82,12 +82,10 @@ def resolve_tokenizer_args( *args, runner_type: "RunnerType" = "generate", tokenizer_mode: str = "auto", - trust_remote_code: bool = False, - revision: str | None = None, - download_dir: str | None = None, **kwargs, ): - tokenizer_kwargs = kwargs + revision: str | None = kwargs.get("revision") + download_dir: str | None = kwargs.get("download_dir") if envs.VLLM_USE_MODELSCOPE: # download model from ModelScope hub, @@ -115,7 +113,7 @@ def resolve_tokenizer_args( # Separate model folder from file path for GGUF models if is_gguf(tokenizer_name): if check_gguf_file(tokenizer_name): - tokenizer_kwargs["gguf_file"] = Path(tokenizer_name).name + kwargs["gguf_file"] = Path(tokenizer_name).name tokenizer_name = Path(tokenizer_name).parent elif is_remote_gguf(tokenizer_name): tokenizer_name, quant_type = split_remote_gguf(tokenizer_name) @@ -125,22 +123,22 @@ def resolve_tokenizer_args( quant_type, revision=revision, ) - tokenizer_kwargs["gguf_file"] = gguf_file + kwargs["gguf_file"] = gguf_file - if "truncation_side" not in tokenizer_kwargs: + if "truncation_side" not in kwargs: if runner_type == "generate" or runner_type == "draft": - tokenizer_kwargs["truncation_side"] = "left" + kwargs["truncation_side"] = "left" elif runner_type == "pooling": - tokenizer_kwargs["truncation_side"] = "right" + kwargs["truncation_side"] = "right" else: assert_never(runner_type) if tokenizer_mode == "slow": - if tokenizer_kwargs.get("use_fast", False): + if kwargs.get("use_fast", False): raise ValueError("Cannot use the fast tokenizer in slow tokenizer mode.") tokenizer_mode = "hf" - tokenizer_kwargs["use_fast"] = False + kwargs["use_fast"] = False # Try to use official Mistral tokenizer if possible if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"): @@ -157,14 +155,7 @@ def resolve_tokenizer_args( if tokenizer_mode == "auto": tokenizer_mode = "hf" - tokenizer_args = args - tokenizer_kwargs = { - "trust_remote_code": trust_remote_code, - "revision": revision, - **tokenizer_kwargs, - } - - return tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs + return tokenizer_mode, tokenizer_name, args, kwargs cached_resolve_tokenizer_args = lru_cache(resolve_tokenizer_args) @@ -194,15 +185,13 @@ def get_tokenizer( **kwargs, ) -> _T: """Gets a tokenizer for the given model name via HuggingFace or ModelScope.""" - tokenizer_mode, tokenizer_name, tokenizer_args, tokenizer_kwargs = ( - cached_resolve_tokenizer_args( - tokenizer_name, - *args, - trust_remote_code=trust_remote_code, - revision=revision, - download_dir=download_dir, - **kwargs, - ) + tokenizer_mode, tokenizer_name, args, kwargs = cached_resolve_tokenizer_args( + tokenizer_name, + *args, + trust_remote_code=trust_remote_code, + revision=revision, + download_dir=download_dir, + **kwargs, ) if tokenizer_cls == TokenizerLike: @@ -210,9 +199,7 @@ def get_tokenizer( else: tokenizer_cls_ = tokenizer_cls - tokenizer = tokenizer_cls_.from_pretrained( - tokenizer_name, *tokenizer_args, **tokenizer_kwargs - ) + tokenizer = tokenizer_cls_.from_pretrained(tokenizer_name, *args, **kwargs) if not tokenizer.is_fast: logger.warning( "Using a slow tokenizer. This might cause a significant " From ddd9590c7dd1baf6b92efb0173f3fcf0aee20d9e Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:21:12 +0000 Subject: [PATCH 32/65] Remove `RequestPrompt` Signed-off-by: DarkLight1337 --- .../openai/test_serving_responses.py | 6 +- vllm/entrypoints/openai/serving_chat.py | 23 ++---- vllm/entrypoints/openai/serving_engine.py | 73 +++++-------------- vllm/entrypoints/openai/serving_responses.py | 22 +++--- vllm/entrypoints/pooling/classify/serving.py | 6 +- vllm/entrypoints/pooling/embed/serving.py | 49 ++++--------- vllm/entrypoints/pooling/pooling/serving.py | 6 +- vllm/entrypoints/renderer.py | 31 ++++---- vllm/entrypoints/serve/disagg/serving.py | 4 +- vllm/entrypoints/serve/tokenize/serving.py | 6 +- 10 files changed, 74 insertions(+), 152 deletions(-) diff --git a/tests/entrypoints/openai/test_serving_responses.py b/tests/entrypoints/openai/test_serving_responses.py index cf00f0a04224..7d03dccec30d 100644 --- a/tests/entrypoints/openai/test_serving_responses.py +++ b/tests/entrypoints/openai/test_serving_responses.py @@ -21,7 +21,7 @@ extract_tool_types, ) from vllm.entrypoints.tool_server import ToolServer -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt +from vllm.inputs.data import TokensPrompt class MockConversationContext(ConversationContext): @@ -237,7 +237,7 @@ def test_validate_generator_input(self, serving_responses_instance): """Test _validate_generator_input with valid prompt length""" # Create an engine prompt with valid length (less than max_model_len) valid_prompt_token_ids = list(range(5)) # 5 tokens < 100 max_model_len - engine_prompt = EngineTokensPrompt(prompt_token_ids=valid_prompt_token_ids) + engine_prompt = TokensPrompt(prompt_token_ids=valid_prompt_token_ids) # Call the method result = serving_responses_instance._validate_generator_input(engine_prompt) @@ -247,7 +247,7 @@ def test_validate_generator_input(self, serving_responses_instance): # create an invalid engine prompt invalid_prompt_token_ids = list(range(200)) # 100 tokens >= 100 max_model_len - engine_prompt = EngineTokensPrompt(prompt_token_ids=invalid_prompt_token_ids) + engine_prompt = TokensPrompt(prompt_token_ids=invalid_prompt_token_ids) # Call the method result = serving_responses_instance._validate_generator_input(engine_prompt) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 117a6ae1072f..17ea659c0736 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -57,7 +57,7 @@ from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall from vllm.entrypoints.openai.utils import maybe_filter_parallel_tool_calls from vllm.entrypoints.utils import get_max_tokens, should_include_usage -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt +from vllm.inputs.data import TokensPrompt from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.outputs import CompletionOutput, RequestOutput @@ -231,11 +231,8 @@ async def create_chat_completion( ) if error_check_ret is not None: return error_check_ret - ( - conversation, - request_prompts, - engine_prompts, - ) = await self._preprocess_chat( + + conversation, engine_prompts = await self._preprocess_chat( request, renderer, request.messages, @@ -251,11 +248,7 @@ async def create_chat_completion( ) else: # For GPT-OSS. - ( - conversation, - request_prompts, - engine_prompts, - ) = self._make_request_with_harmony(request) + conversation, engine_prompts = self._make_request_with_harmony(request) except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e: logger.exception("Error in preprocessing prompt inputs") return self.create_error_response(f"{e} {e.__cause__}") @@ -275,7 +268,7 @@ async def create_chat_completion( generators: list[AsyncGenerator[RequestOutput, None]] = [] try: for i, engine_prompt in enumerate(engine_prompts): - prompt_text, _, _ = self._get_prompt_components(request_prompts[i]) + prompt_text, _, _ = self._get_prompt_components(engine_prompt) # If we are creating sub requests for multiple prompts, ensure that they # have unique request ids. sub_request_id = ( @@ -310,7 +303,7 @@ async def create_chat_completion( self._log_inputs( sub_request_id, - request_prompts[i], + engine_prompt, params=sampling_params, lora_request=lora_request, ) @@ -1804,10 +1797,10 @@ def _make_request_with_harmony( # Render prompt token ids. prompt_token_ids = render_for_completion(messages) - engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids) + engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) # Add cache_salt if provided in the request if request.cache_salt is not None: engine_prompt["cache_salt"] = request.cache_salt - return messages, [prompt_token_ids], [engine_prompt] + return messages, [engine_prompt] diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index e6382a454cf1..3cc8ed2ee98c 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -5,7 +5,7 @@ import sys import time import traceback -from collections.abc import AsyncGenerator, Callable, Iterable, Mapping, Sequence +from collections.abc import AsyncGenerator, Callable, Iterable, Mapping from dataclasses import dataclass, field from http import HTTPStatus from typing import Any, ClassVar, Generic, TypeAlias, TypeVar @@ -15,7 +15,6 @@ from fastapi import Request from pydantic import ConfigDict, TypeAdapter from starlette.datastructures import Headers -from typing_extensions import TypeIs from vllm.entrypoints.context import ( HarmonyContext, @@ -93,9 +92,7 @@ ) from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse from vllm.entrypoints.utils import _validate_truncation_size -from vllm.inputs.data import PromptType, SingletonPrompt -from vllm.inputs.data import TextPrompt as EngineTextPrompt -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt +from vllm.inputs.data import PromptType, TextPrompt, TokensPrompt from vllm.inputs.parse import ( PromptComponents, get_prompt_components, @@ -174,25 +171,6 @@ class EmbedsPrompt(TypedDict): prompt_embeds: torch.Tensor -RequestPrompt: TypeAlias = list[int] | TextTokensPrompt | EmbedsPrompt | SingletonPrompt - - -def is_text_tokens_prompt(prompt: RequestPrompt) -> TypeIs[TextTokensPrompt]: - return ( - isinstance(prompt, dict) - and "prompt_token_ids" in prompt - and "prompt_embeds" not in prompt - ) - - -def is_embeds_prompt(prompt: RequestPrompt) -> TypeIs[EmbedsPrompt]: - return ( - isinstance(prompt, dict) - and "prompt_token_ids" not in prompt - and "prompt_embeds" in prompt - ) - - RequestT = TypeVar("RequestT", bound=AnyRequest) @@ -203,8 +181,7 @@ class RequestProcessingMixin: handling prompt preparation and engine input. """ - request_prompts: Sequence[RequestPrompt] | None = field(default_factory=list) - engine_prompts: list[EngineTokensPrompt] | None = field(default_factory=list) + engine_prompts: list[TokensPrompt] | None = field(default_factory=list) @dataclass(kw_only=True) @@ -398,7 +375,7 @@ async def beam_search( prompts_batch, lora_req_batch = zip( *[ ( - EngineTokensPrompt( + TokensPrompt( prompt_token_ids=beam.tokens, multi_modal_data=beam.multi_modal_data, mm_processor_kwargs=beam.mm_processor_kwargs, @@ -1079,11 +1056,7 @@ async def _preprocess_chat( chat_template_kwargs: dict[str, Any] | None = None, tool_parser: Callable[[TokenizerLike], ToolParser] | None = None, add_special_tokens: bool = False, - ) -> tuple[ - list[ConversationMessage], - Sequence[RequestPrompt], - list[EngineTokensPrompt], - ]: + ) -> tuple[list[ConversationMessage], list[TokensPrompt]]: chat_template_kwargs = { "chat_template": chat_template, "add_generation_prompt": add_generation_prompt, @@ -1093,7 +1066,7 @@ async def _preprocess_chat( **(chat_template_kwargs or {}), } - engine_prompt: EngineTokensPrompt | EngineTextPrompt + engine_prompt: TokensPrompt | TextPrompt conversation, engine_prompt = await renderer.render_messages_async( messages, chat_template_content_format=chat_template_content_format, @@ -1107,7 +1080,7 @@ async def _preprocess_chat( engine_prompt["prompt"], add_special_tokens=add_special_tokens, ) - engine_prompt = EngineTokensPrompt(**tokenized_prompt) + engine_prompt = TokensPrompt(**tokenized_prompt) if request.mm_processor_kwargs is not None: engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs @@ -1132,7 +1105,7 @@ async def _preprocess_chat( tokenizer = renderer.get_tokenizer() request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore - return conversation, [engine_prompt], [engine_prompt] # type: ignore[list-item] + return conversation, [engine_prompt] # type: ignore[list-item] async def _process_inputs( self, @@ -1175,7 +1148,7 @@ async def _render_next_turn( request_input=messages, ) - _, request_prompts, engine_prompts = await self._preprocess_chat( + _, engine_prompts = await self._preprocess_chat( request, renderer, new_messages, @@ -1184,20 +1157,20 @@ async def _render_next_turn( chat_template=chat_template, chat_template_content_format=chat_template_content_format, ) - return request_prompts, engine_prompts + return engine_prompts async def _generate_with_builtin_tools( self, request_id: str, - request_prompt: RequestPrompt, - engine_prompt: EngineTokensPrompt, + engine_prompt: TokensPrompt, sampling_params: SamplingParams, context: ConversationContext, lora_request: LoRARequest | None = None, priority: int = 0, **kwargs, ): - prompt_text, _, _ = self._get_prompt_components(request_prompt) + prompt_text, _, _ = self._get_prompt_components(engine_prompt) + orig_priority = priority sub_request = 0 while True: @@ -1205,7 +1178,7 @@ async def _generate_with_builtin_tools( sub_request_id = f"{request_id}_{sub_request}" self._log_inputs( sub_request_id, - request_prompt, + engine_prompt, params=sampling_params, lora_request=lora_request, ) @@ -1250,10 +1223,9 @@ async def _generate_with_builtin_tools( # Render the next prompt token ids. if isinstance(context, (HarmonyContext, StreamingHarmonyContext)): prompt_token_ids = context.render_for_completion() - engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids) - request_prompt = prompt_token_ids + engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) elif isinstance(context, ParsableContext): - request_prompts, engine_prompts = await self._render_next_turn( + engine_prompts = await self._render_next_turn( context.request, context.renderer, context.parser.response_messages, @@ -1263,7 +1235,6 @@ async def _generate_with_builtin_tools( context.chat_template_content_format, ) engine_prompt = engine_prompts[0] - request_prompt = request_prompts[0] # Update the sampling params. sampling_params.max_tokens = self.max_model_len - len( @@ -1273,19 +1244,13 @@ async def _generate_with_builtin_tools( priority = orig_priority - 1 sub_request += 1 - def _get_prompt_components( - self, - prompt: RequestPrompt | PromptType, - ) -> PromptComponents: - if isinstance(prompt, list): - return PromptComponents(token_ids=prompt) - - return get_prompt_components(prompt) # type: ignore[arg-type] + def _get_prompt_components(self, prompt: PromptType) -> PromptComponents: + return get_prompt_components(prompt) def _log_inputs( self, request_id: str, - inputs: RequestPrompt | PromptType, + inputs: PromptType, params: SamplingParams | PoolingParams | BeamSearchParams | None, lora_request: LoRARequest | None, ) -> None: diff --git a/vllm/entrypoints/openai/serving_responses.py b/vllm/entrypoints/openai/serving_responses.py index 1bb0c76f8e8e..a78df87af05f 100644 --- a/vllm/entrypoints/openai/serving_responses.py +++ b/vllm/entrypoints/openai/serving_responses.py @@ -103,7 +103,7 @@ make_response_output_items_from_parsable_context, ) from vllm.entrypoints.tool_server import ToolServer -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt +from vllm.inputs.data import TokensPrompt from vllm.logger import init_logger from vllm.logprobs import Logprob as SampleLogprob from vllm.logprobs import SampleLogprobs @@ -255,7 +255,7 @@ def __init__( self.tool_server = tool_server def _validate_generator_input( - self, engine_prompt: EngineTokensPrompt + self, engine_prompt: TokensPrompt ) -> ErrorResponse | None: """Add validations to the input to the generator here.""" if self.max_model_len <= len(engine_prompt["prompt_token_ids"]): @@ -351,11 +351,11 @@ async def create_responses( tokenizer = renderer.get_tokenizer() if self.use_harmony: - messages, request_prompts, engine_prompts = ( - self._make_request_with_harmony(request, prev_response) + messages, engine_prompts = self._make_request_with_harmony( + request, prev_response ) else: - messages, request_prompts, engine_prompts = await self._make_request( + messages, engine_prompts = await self._make_request( request, prev_response, renderer ) @@ -391,7 +391,7 @@ async def create_responses( assert len(builtin_tool_list) == 0 available_tools = [] try: - for i, engine_prompt in enumerate(engine_prompts): + for engine_prompt in engine_prompts: maybe_error = self._validate_generator_input(engine_prompt) if maybe_error is not None: return maybe_error @@ -447,7 +447,6 @@ async def create_responses( ) generator = self._generate_with_builtin_tools( request_id=request.request_id, - request_prompt=request_prompts[i], engine_prompt=engine_prompt, sampling_params=sampling_params, context=context, @@ -560,7 +559,7 @@ async def _make_request( prev_msg=self.msg_store.get(prev_response.id) if prev_response else None, prev_response_output=prev_response.output if prev_response else None, ) - _, request_prompts, engine_prompts = await self._preprocess_chat( + _, engine_prompts = await self._preprocess_chat( request, renderer, messages, @@ -569,7 +568,7 @@ async def _make_request( chat_template=self.chat_template, chat_template_content_format=self.chat_template_content_format, ) - return messages, request_prompts, engine_prompts + return messages, engine_prompts def _make_request_with_harmony( self, @@ -580,15 +579,16 @@ def _make_request_with_harmony( raise NotImplementedError( "Only 'auto' tool_choice is supported in response API with Harmony" ) + messages = self._construct_input_messages_with_harmony(request, prev_response) prompt_token_ids = render_for_completion(messages) - engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_token_ids) + engine_prompt = TokensPrompt(prompt_token_ids=prompt_token_ids) # Add cache_salt if provided in the request if request.cache_salt is not None: engine_prompt["cache_salt"] = request.cache_salt - return messages, [prompt_token_ids], [engine_prompt] + return messages, [engine_prompt] async def _initialize_tool_sessions( self, diff --git a/vllm/entrypoints/pooling/classify/serving.py b/vllm/entrypoints/pooling/classify/serving.py index 5e6d1e7781e5..9ddf9b7bb2b7 100644 --- a/vllm/entrypoints/pooling/classify/serving.py +++ b/vllm/entrypoints/pooling/classify/serving.py @@ -70,11 +70,7 @@ async def _preprocess( if ret: return ret - ( - _, - _, - engine_prompts, - ) = await self._preprocess_chat( + _, engine_prompts = await self._preprocess_chat( cast(ChatCompletionRequest, chat_request), self.renderer, messages, diff --git a/vllm/entrypoints/pooling/embed/serving.py b/vllm/entrypoints/pooling/embed/serving.py index 8d10be5f5a9a..3b25236be71d 100644 --- a/vllm/entrypoints/pooling/embed/serving.py +++ b/vllm/entrypoints/pooling/embed/serving.py @@ -32,7 +32,7 @@ EmbeddingResponseData, ) from vllm.entrypoints.renderer import RenderConfig -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt +from vllm.inputs.data import TokensPrompt from vllm.logger import init_logger from vllm.outputs import ( EmbeddingRequestOutput, @@ -80,11 +80,7 @@ async def _preprocess( ctx.lora_request = self._maybe_get_adapters(ctx.request) if isinstance(ctx.request, EmbeddingChatRequest): - ( - _, - _, - ctx.engine_prompts, - ) = await self._preprocess_chat( + _, ctx.engine_prompts = await self._preprocess_chat( ctx.request, self.renderer, ctx.request.messages, @@ -201,14 +197,13 @@ def _should_use_chunked_processing(self, request) -> bool: async def _process_chunked_request( self, ctx: EmbeddingServeContext, - original_prompt: TextTokensPrompt, + token_ids: list[int], pooling_params, trace_headers, prompt_idx: int, ) -> list[AsyncGenerator[PoolingRequestOutput, None]]: """Process a single prompt using chunked processing.""" generators: list[AsyncGenerator[PoolingRequestOutput, None]] = [] - token_ids = original_prompt["prompt_token_ids"] # Split into chunks using max_position_embeddings max_pos_embeddings = self._get_max_position_embeddings() @@ -220,18 +215,12 @@ async def _process_chunked_request( chunk_request_id = f"{ctx.request_id}-prompt-{prompt_idx}-chunk-{chunk_idx}" # Create engine prompt for this chunk - chunk_engine_prompt = EngineTokensPrompt(prompt_token_ids=chunk_tokens) - - # Create chunk request prompt for logging - chunk_text = "" - chunk_request_prompt = TextTokensPrompt( - prompt=chunk_text, prompt_token_ids=chunk_tokens - ) + chunk_engine_prompt = TokensPrompt(prompt_token_ids=chunk_tokens) # Log the chunk self._log_inputs( chunk_request_id, - chunk_request_prompt, + chunk_engine_prompt, params=pooling_params, lora_request=ctx.lora_request, ) @@ -325,18 +314,10 @@ def _validate_input( # For other request types, use the parent's implementation return super()._validate_input(request, input_ids, input_text) - def _is_text_tokens_prompt(self, prompt) -> bool: - """Check if a prompt is a TextTokensPrompt (has prompt_token_ids).""" - return ( - isinstance(prompt, dict) - and "prompt_token_ids" in prompt - and "prompt_embeds" not in prompt - ) - async def _create_single_prompt_generator( self, ctx: EmbeddingServeContext, - engine_prompt: EngineTokensPrompt, + engine_prompt: TokensPrompt, pooling_params: PoolingParams, trace_headers: Mapping[str, str] | None, prompt_index: int, @@ -405,14 +386,11 @@ async def _prepare_generators( for i, engine_prompt in enumerate(ctx.engine_prompts): # Check if this specific prompt needs chunked processing - if self._is_text_tokens_prompt(engine_prompt): - # Cast to TextTokensPrompt since we've verified - # prompt_token_ids - text_tokens_prompt = cast(TextTokensPrompt, engine_prompt) - if len(text_tokens_prompt["prompt_token_ids"]) > max_pos_embeddings: + if "prompt_token_ids" in engine_prompt: # noqa: SIM102 + if len(engine_prompt["prompt_token_ids"]) > max_pos_embeddings: # Use chunked processing for this prompt chunk_generators = await self._process_chunked_request( - ctx, text_tokens_prompt, pooling_params, trace_headers, i + ctx, engine_prompt, pooling_params, trace_headers, i ) generators.extend(chunk_generators) continue @@ -570,14 +548,13 @@ async def _collect_batch( # Get original prompt token IDs for this prompt original_prompt = ctx.engine_prompts[prompt_idx] - if not self._is_text_tokens_prompt(original_prompt): + if "prompt_token_ids" not in original_prompt: return self.create_error_response( - f"Chunked prompt {prompt_idx} is not a TextTokensPrompt" + f"Chunked prompt {prompt_idx} does not contain " + "token IDs" ) - original_token_ids = cast(TextTokensPrompt, original_prompt)[ - "prompt_token_ids" - ] + original_token_ids = original_prompt["prompt_token_ids"] pooling_request_output = PoolingRequestOutput( request_id=aggregator["request_id"], diff --git a/vllm/entrypoints/pooling/pooling/serving.py b/vllm/entrypoints/pooling/pooling/serving.py index 44c27fd8522e..84546d69a06e 100644 --- a/vllm/entrypoints/pooling/pooling/serving.py +++ b/vllm/entrypoints/pooling/pooling/serving.py @@ -131,11 +131,7 @@ async def create_pooling( ) if error_check_ret is not None: return error_check_ret - ( - _, - _, - engine_prompts, - ) = await self._preprocess_chat( + _, engine_prompts = await self._preprocess_chat( request, self.renderer, request.messages, diff --git a/vllm/entrypoints/renderer.py b/vllm/entrypoints/renderer.py index f31b309b8ca4..aa90db5ec987 100644 --- a/vllm/entrypoints/renderer.py +++ b/vllm/entrypoints/renderer.py @@ -13,8 +13,7 @@ from vllm.config import ModelConfig from vllm.inputs.data import EmbedsPrompt as EngineEmbedsPrompt -from vllm.inputs.data import TextPrompt as EngineTextPrompt -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt +from vllm.inputs.data import TextPrompt, TokensPrompt from vllm.inputs.parse import get_prompt_components, parse_raw_prompts from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import AsyncMicrobatchTokenizer @@ -97,7 +96,7 @@ async def render_prompt( *, prompt_or_prompts: str | list[str] | list[int] | list[list[int]], config: RenderConfig, - ) -> list[EngineTokensPrompt]: + ) -> list[TokensPrompt]: """ Convert text or token inputs into engine-ready TokensPrompt objects. @@ -115,7 +114,7 @@ async def render_prompt( (e.g., tokenization and length handling). Returns: - list[EngineTokensPrompt]: Engine-ready token prompts. + list[TokensPrompt]: Engine-ready token prompts. Raises: ValueError: If input formats are invalid or length limits exceeded. @@ -129,7 +128,7 @@ async def render_prompt_and_embeds( prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None, prompt_embeds: bytes | list[bytes] | None = None, config: RenderConfig, - ) -> list[EngineTokensPrompt | EngineEmbedsPrompt]: + ) -> list[TokensPrompt | EngineEmbedsPrompt]: """ Convert text/token and/or base64-encoded embeddings inputs into engine-ready prompt objects using a unified RenderConfig. @@ -146,7 +145,7 @@ async def render_prompt_and_embeds( (e.g., tokenization and length handling). Returns: - list[Union[EngineTokensPrompt, EngineEmbedsPrompt]]: + list[Union[TokensPrompt, EngineEmbedsPrompt]]: Engine-ready prompt objects. Raises: @@ -213,7 +212,7 @@ async def render_prompt( *, prompt_or_prompts: str | list[str] | list[int] | list[list[int]], config: RenderConfig, - ) -> list[EngineTokensPrompt]: + ) -> list[TokensPrompt]: """Implementation of prompt rendering for completion-style requests. Uses async tokenizer pooling for improved performance. See base class @@ -240,7 +239,7 @@ async def render_prompt_and_embeds( prompt_or_prompts: str | list[str] | list[int] | list[list[int]] | None = None, prompt_embeds: bytes | list[bytes] | None = None, config: RenderConfig, - ) -> list[EngineTokensPrompt | EngineEmbedsPrompt]: + ) -> list[TokensPrompt | EngineEmbedsPrompt]: """ Render text/token prompts and/or precomputed embedding prompts. At least one of `prompt_or_prompts` or `prompt_embeds` must be provided. @@ -249,7 +248,7 @@ async def render_prompt_and_embeds( if truncate_prompt_tokens == 0: return [] - rendered: list[EngineTokensPrompt | EngineEmbedsPrompt] = [] + rendered: list[TokensPrompt | EngineEmbedsPrompt] = [] if prompt_embeds is not None: rendered.extend( @@ -281,10 +280,10 @@ def _maybe_apply_truncation( async def _create_prompt( self, - prompt_input: EngineTextPrompt | EngineTokensPrompt, + prompt_input: TextPrompt | TokensPrompt, config: RenderConfig, truncate_prompt_tokens: int | None, - ) -> EngineTokensPrompt: + ) -> TokensPrompt: prompt, prompt_token_ids, _ = get_prompt_components(prompt_input) if prompt_token_ids is not None: @@ -317,7 +316,7 @@ async def _create_prompt_from_text( truncate_prompt_tokens: int | None, add_special_tokens: bool, cache_salt: str | None, - ) -> EngineTokensPrompt: + ) -> TokensPrompt: """Tokenize text input asynchronously.""" async_tokenizer = self._get_async_tokenizer() @@ -350,7 +349,7 @@ async def _create_prompt_from_token_ids( truncate_prompt_tokens: int | None, cache_salt: str | None, needs_detokenization: bool | None = False, - ) -> EngineTokensPrompt: + ) -> TokensPrompt: """Optionally detokenize token IDs and build a tokens prompt.""" token_ids = self._maybe_apply_truncation(token_ids, truncate_prompt_tokens) @@ -392,8 +391,8 @@ def _create_tokens_prompt( max_length: int | None = None, cache_salt: str | None = None, prompt: str | None = None, - ) -> EngineTokensPrompt: - """Create validated EngineTokensPrompt.""" + ) -> TokensPrompt: + """Create validated TokensPrompt.""" if max_length is not None and len(token_ids) > max_length: raise ValueError( f"This model's maximum context length is {max_length} tokens. " @@ -401,7 +400,7 @@ def _create_tokens_prompt( "Please reduce the length of the input messages." ) - tokens_prompt = EngineTokensPrompt(prompt_token_ids=token_ids) + tokens_prompt = TokensPrompt(prompt_token_ids=token_ids) if cache_salt is not None: tokens_prompt["cache_salt"] = cache_salt if prompt is not None: diff --git a/vllm/entrypoints/serve/disagg/serving.py b/vllm/entrypoints/serve/disagg/serving.py index 5c1d17156a90..08841c865c13 100644 --- a/vllm/entrypoints/serve/disagg/serving.py +++ b/vllm/entrypoints/serve/disagg/serving.py @@ -27,7 +27,7 @@ GenerateResponse, GenerateResponseChoice, ) -from vllm.inputs.data import TokensPrompt as EngineTokensPrompt +from vllm.inputs.data import TokensPrompt from vllm.logger import init_logger from vllm.logprobs import Logprob from vllm.outputs import RequestOutput @@ -99,7 +99,7 @@ async def serve_tokens( # TODO(NickLucche): Change to EngineCoreRequest once Renderer work is # completed - engine_prompt = EngineTokensPrompt(prompt_token_ids=request.token_ids) + engine_prompt = TokensPrompt(prompt_token_ids=request.token_ids) if request.features is not None: engine_prompt["multi_modal_data"] = None diff --git a/vllm/entrypoints/serve/tokenize/serving.py b/vllm/entrypoints/serve/tokenize/serving.py index 29eb2ec609e6..ac3205bf3fb5 100644 --- a/vllm/entrypoints/serve/tokenize/serving.py +++ b/vllm/entrypoints/serve/tokenize/serving.py @@ -77,11 +77,7 @@ async def create_tokenize( ) if error_check_ret is not None: return error_check_ret - ( - _, - _, - engine_prompts, - ) = await self._preprocess_chat( + _, engine_prompts = await self._preprocess_chat( request, self.renderer, request.messages, From ed3fd863222865fd342bc79c582fbad4642d569a Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:27:08 +0000 Subject: [PATCH 33/65] Simplify Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 33 ++++++++--------------- vllm/entrypoints/pooling/embed/serving.py | 5 ++-- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 3cc8ed2ee98c..00c4cff3f08f 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -11,7 +11,6 @@ from typing import Any, ClassVar, Generic, TypeAlias, TypeVar import numpy as np -import torch from fastapi import Request from pydantic import ConfigDict, TypeAdapter from starlette.datastructures import Headers @@ -49,9 +48,9 @@ ) if sys.version_info >= (3, 12): - from typing import TypedDict + pass else: - from typing_extensions import TypedDict + pass from openai.types.responses import ( ToolChoiceFunction, @@ -162,15 +161,6 @@ ) -class TextTokensPrompt(TypedDict): - prompt: str - prompt_token_ids: list[int] - - -class EmbedsPrompt(TypedDict): - prompt_embeds: torch.Tensor - - RequestT = TypeVar("RequestT", bound=AnyRequest) @@ -845,7 +835,7 @@ async def _normalize_prompt_text_to_input( prompt: str, tokenizer: TokenizerLike, add_special_tokens: bool, - ) -> TextTokensPrompt: + ) -> TokensPrompt: async_tokenizer = self._get_async_tokenizer(tokenizer) if ( @@ -886,7 +876,7 @@ async def _normalize_prompt_tokens_to_input( request: AnyRequest, prompt_ids: list[int], tokenizer: TokenizerLike | None, - ) -> TextTokensPrompt: + ) -> TokensPrompt: truncate_prompt_tokens = getattr(request, "truncate_prompt_tokens", None) if truncate_prompt_tokens is None: @@ -909,7 +899,7 @@ def _validate_input( request: AnyRequest, input_ids: list[int], input_text: str, - ) -> TextTokensPrompt: + ) -> TokensPrompt: token_num = len(input_ids) # Note: EmbeddingRequest, ClassificationRequest, @@ -940,7 +930,7 @@ def _validate_input( f"{token_num} tokens in the input for {operation}. " f"Please reduce the length of the input." ) - return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids) + return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids) # Note: TokenizeRequest and DetokenizeRequest doesn't have max_tokens # and does not require model context length validation @@ -948,7 +938,7 @@ def _validate_input( request, (TokenizeCompletionRequest, TokenizeChatRequest, DetokenizeRequest), ): - return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids) + return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids) # chat completion endpoint supports max_completion_tokens if isinstance(request, ChatCompletionRequest): @@ -976,7 +966,7 @@ def _validate_input( f" - {token_num})." ) - return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids) + return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids) async def _tokenize_prompt_input_async( self, @@ -984,7 +974,7 @@ async def _tokenize_prompt_input_async( tokenizer: TokenizerLike, prompt_input: str | list[int], add_special_tokens: bool = True, - ) -> TextTokensPrompt: + ) -> TokensPrompt: """ A simpler implementation that tokenizes a single prompt input. """ @@ -1003,7 +993,7 @@ async def _tokenize_prompt_inputs_async( tokenizer: TokenizerLike, prompt_inputs: Iterable[str | list[int]], add_special_tokens: bool = True, - ) -> AsyncGenerator[TextTokensPrompt, None]: + ) -> AsyncGenerator[TokensPrompt, None]: """ A simpler implementation that tokenizes multiple prompt inputs. """ @@ -1074,13 +1064,12 @@ async def _preprocess_chat( ) if "prompt_token_ids" not in engine_prompt: - tokenized_prompt = await self._tokenize_prompt_input_async( + engine_prompt = await self._tokenize_prompt_input_async( request, renderer.get_tokenizer(), engine_prompt["prompt"], add_special_tokens=add_special_tokens, ) - engine_prompt = TokensPrompt(**tokenized_prompt) if request.mm_processor_kwargs is not None: engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs diff --git a/vllm/entrypoints/pooling/embed/serving.py b/vllm/entrypoints/pooling/embed/serving.py index 3b25236be71d..d7b322e7ba38 100644 --- a/vllm/entrypoints/pooling/embed/serving.py +++ b/vllm/entrypoints/pooling/embed/serving.py @@ -20,7 +20,6 @@ EmbeddingServeContext, OpenAIServing, ServeContext, - TextTokensPrompt, ) from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.pooling.embed.protocol import ( @@ -244,7 +243,7 @@ def _validate_input( request, input_ids: list[int], input_text: str, - ) -> TextTokensPrompt: + ) -> TokensPrompt: """Override to support chunked processing for embedding requests.""" token_num = len(input_ids) @@ -309,7 +308,7 @@ def _validate_input( ) ) - return TextTokensPrompt(prompt=input_text, prompt_token_ids=input_ids) + return TokensPrompt(prompt=input_text, prompt_token_ids=input_ids) # For other request types, use the parent's implementation return super()._validate_input(request, input_ids, input_text) From 10814a7a5ada97463f770819628df6687888ca16 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:37:57 +0000 Subject: [PATCH 34/65] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 00c4cff3f08f..cd3ba242f3ec 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -91,7 +91,7 @@ ) from vllm.entrypoints.serve.disagg.protocol import GenerateRequest, GenerateResponse from vllm.entrypoints.utils import _validate_truncation_size -from vllm.inputs.data import PromptType, TextPrompt, TokensPrompt +from vllm.inputs.data import PromptType, TokensPrompt from vllm.inputs.parse import ( PromptComponents, get_prompt_components, @@ -1056,10 +1056,15 @@ async def _preprocess_chat( **(chat_template_kwargs or {}), } - engine_prompt: TokensPrompt | TextPrompt + # For now, use the async tokenizer for chat template tokenization if possible + # Later we can move the async tokenizer into the renderer so we can return both + # text and token IDs from render_messages_async. + from vllm.tokenizers.mistral import MistralTokenizer + conversation, engine_prompt = await renderer.render_messages_async( messages, chat_template_content_format=chat_template_content_format, + tokenize=isinstance(renderer.tokenizer, MistralTokenizer), **chat_template_kwargs, ) From 67e5f7b9233e3317d84f6626b0943fa10ce3371e Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:43:00 +0000 Subject: [PATCH 35/65] Comment Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index cd3ba242f3ec..9bf3f7d59db0 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1056,9 +1056,10 @@ async def _preprocess_chat( **(chat_template_kwargs or {}), } - # For now, use the async tokenizer for chat template tokenization if possible + # For now, use the async tokenizer for chat template tokenization if possible. # Later we can move the async tokenizer into the renderer so we can return both - # text and token IDs from render_messages_async. + # text and token IDs in the same prompt from `render_messages_async` which is + # used for logging and `enable_response_messages`. from vllm.tokenizers.mistral import MistralTokenizer conversation, engine_prompt = await renderer.render_messages_async( From 88700ee206429608398bf10b06e37558c2e02f95 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:46:16 +0000 Subject: [PATCH 36/65] Reword Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 9bf3f7d59db0..ee885d6074ef 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1056,10 +1056,10 @@ async def _preprocess_chat( **(chat_template_kwargs or {}), } - # For now, use the async tokenizer for chat template tokenization if possible. - # Later we can move the async tokenizer into the renderer so we can return both - # text and token IDs in the same prompt from `render_messages_async` which is - # used for logging and `enable_response_messages`. + # Use the async tokenizer in `OpenAIServing` if possible. + # Later we can move it into the renderer so that we can return both + # text and token IDs in the same prompt from `render_messages_async` + # which is used for logging and `enable_response_messages`. from vllm.tokenizers.mistral import MistralTokenizer conversation, engine_prompt = await renderer.render_messages_async( From e8787f7e649519b3a85d5f4f807cd2538b856e4e Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:53:46 +0000 Subject: [PATCH 37/65] Unnecessary lazy import Signed-off-by: DarkLight1337 --- vllm/tokenizers/hf.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/vllm/tokenizers/hf.py b/vllm/tokenizers/hf.py index c455d80a3fbc..a7b565dca5d8 100644 --- a/vllm/tokenizers/hf.py +++ b/vllm/tokenizers/hf.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import TypeAlias -from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast +from transformers import AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast from vllm.transformers_utils.config import get_sentence_transformer_tokenizer_config @@ -75,8 +75,6 @@ def from_pretrained( download_dir: str | None = None, **kwargs, ) -> HfTokenizer: - from transformers import AutoTokenizer - try: tokenizer = AutoTokenizer.from_pretrained( path_or_repo_id, From 9dca5ebb64e34b2ffa7e7e02a52d3e25072561ec Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:56:10 +0000 Subject: [PATCH 38/65] Simplify Signed-off-by: DarkLight1337 --- tests/tokenizers_/test_registry.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/tokenizers_/test_registry.py b/tests/tokenizers_/test_registry.py index 0c718edffff4..546f38b078dd 100644 --- a/tests/tokenizers_/test_registry.py +++ b/tests/tokenizers_/test_registry.py @@ -49,10 +49,8 @@ def is_fast(self) -> bool: @pytest.mark.parametrize("runner_type", ["generate", "pooling"]) def test_resolve_tokenizer_args_idempotent(runner_type): - tokenizer_name = "facebook/opt-125m" - tokenizer_mode, tokenizer_name, args, kwargs = resolve_tokenizer_args( - tokenizer_name, + "facebook/opt-125m", runner_type=runner_type, ) From 854ac7717fea2e39cbe9bafa8d85cb581784123e Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 06:58:06 +0000 Subject: [PATCH 39/65] Reduce diff Signed-off-by: DarkLight1337 --- tests/entrypoints/openai/test_chat_template.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/entrypoints/openai/test_chat_template.py b/tests/entrypoints/openai/test_chat_template.py index 7d912af39424..0762b4bcb696 100644 --- a/tests/entrypoints/openai/test_chat_template.py +++ b/tests/entrypoints/openai/test_chat_template.py @@ -8,7 +8,6 @@ from vllm.entrypoints.openai.protocol import ChatCompletionRequest from vllm.renderers.hf import safe_apply_chat_template from vllm.tokenizers import get_tokenizer -from vllm.tokenizers.hf import CachedHfTokenizer from ...models.registry import HF_EXAMPLE_MODELS from ...utils import VLLM_PATH @@ -125,8 +124,7 @@ def test_get_gen_prompt( # Initialize the tokenizer tokenizer = get_tokenizer( - model_config.tokenizer, - tokenizer_cls=CachedHfTokenizer, + tokenizer_name=model_config.tokenizer, trust_remote_code=model_config.trust_remote_code, ) template_content = load_chat_template(chat_template=template) From 06d1f87e78fd2d610b6a4bbb074f5735b7ae1b0a Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 08:49:15 +0000 Subject: [PATCH 40/65] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/pooling/embed/serving.py | 11 ++++++++--- vllm/entrypoints/serve/disagg/serving.py | 2 +- vllm/entrypoints/serve/tokenize/serving.py | 6 +++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/vllm/entrypoints/pooling/embed/serving.py b/vllm/entrypoints/pooling/embed/serving.py index d7b322e7ba38..18f7928860f9 100644 --- a/vllm/entrypoints/pooling/embed/serving.py +++ b/vllm/entrypoints/pooling/embed/serving.py @@ -385,11 +385,16 @@ async def _prepare_generators( for i, engine_prompt in enumerate(ctx.engine_prompts): # Check if this specific prompt needs chunked processing - if "prompt_token_ids" in engine_prompt: # noqa: SIM102 - if len(engine_prompt["prompt_token_ids"]) > max_pos_embeddings: + if "prompt_token_ids" in engine_prompt: + prompt_token_ids = engine_prompt["prompt_token_ids"] + if len(prompt_token_ids) > max_pos_embeddings: # Use chunked processing for this prompt chunk_generators = await self._process_chunked_request( - ctx, engine_prompt, pooling_params, trace_headers, i + ctx, + prompt_token_ids, + pooling_params, + trace_headers, + i, ) generators.extend(chunk_generators) continue diff --git a/vllm/entrypoints/serve/disagg/serving.py b/vllm/entrypoints/serve/disagg/serving.py index 08841c865c13..1798b174b141 100644 --- a/vllm/entrypoints/serve/disagg/serving.py +++ b/vllm/entrypoints/serve/disagg/serving.py @@ -115,7 +115,7 @@ async def serve_tokens( self._log_inputs( request_id, - request.token_ids, + TokensPrompt(prompt_token_ids=request.token_ids), params=sampling_params, lora_request=lora_request, ) diff --git a/vllm/entrypoints/serve/tokenize/serving.py b/vllm/entrypoints/serve/tokenize/serving.py index ac3205bf3fb5..6700f7a42870 100644 --- a/vllm/entrypoints/serve/tokenize/serving.py +++ b/vllm/entrypoints/serve/tokenize/serving.py @@ -21,6 +21,7 @@ from vllm.entrypoints.openai.serving_engine import OpenAIServing from vllm.entrypoints.openai.serving_models import OpenAIServingModels from vllm.entrypoints.renderer import RenderConfig +from vllm.inputs import TokensPrompt from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike @@ -135,7 +136,10 @@ async def create_detokenize( tokenizer = self.renderer.get_tokenizer() self._log_inputs( - request_id, request.tokens, params=None, lora_request=lora_request + request_id, + TokensPrompt(prompt_token_ids=request.tokens), + params=None, + lora_request=lora_request, ) prompt_input = await self._tokenize_prompt_input_async( From 461dc613d734f7f0be5462ce68037b54c136b581 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 09:44:51 +0000 Subject: [PATCH 41/65] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index ee885d6074ef..74dfb3f5e28c 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1070,12 +1070,15 @@ async def _preprocess_chat( ) if "prompt_token_ids" not in engine_prompt: + extra_data = engine_prompt engine_prompt = await self._tokenize_prompt_input_async( request, renderer.get_tokenizer(), engine_prompt["prompt"], add_special_tokens=add_special_tokens, ) + for k in extra_data: + engine_prompt[k] = extra_data[k] if request.mm_processor_kwargs is not None: engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs From 0a86c2216ecf03c083373906169c9a804fcbaadb Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 11:37:21 +0000 Subject: [PATCH 42/65] Fix mypy and test Signed-off-by: DarkLight1337 --- tests/v1/engine/test_llm_engine.py | 2 +- vllm/entrypoints/openai/serving_engine.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/v1/engine/test_llm_engine.py b/tests/v1/engine/test_llm_engine.py index c1d5f8af7917..7e5196efc873 100644 --- a/tests/v1/engine/test_llm_engine.py +++ b/tests/v1/engine/test_llm_engine.py @@ -224,7 +224,7 @@ def test_skip_tokenizer_initialization(model: str): ) sampling_params = SamplingParams(prompt_logprobs=True, detokenize=True) - with pytest.raises(ValueError, match="cannot pass text prompts when"): + with pytest.raises(ValueError, match="`skip_tokenizer_init=True`"): llm.generate("abc", sampling_params) outputs = llm.generate( diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 74dfb3f5e28c..b684caf1a89f 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1077,8 +1077,9 @@ async def _preprocess_chat( engine_prompt["prompt"], add_special_tokens=add_special_tokens, ) - for k in extra_data: - engine_prompt[k] = extra_data[k] + + # Fill in other keys like MM data + engine_prompt.update(extra_data) # type: ignore if request.mm_processor_kwargs is not None: engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs From 5991f7bea8918107b9f8cf9c74e355bbdfa09606 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 14:13:00 +0000 Subject: [PATCH 43/65] Update tests Signed-off-by: DarkLight1337 --- tests/entrypoints/openai/test_serving_chat.py | 8 ++++++++ tests/entrypoints/openai/test_serving_engine.py | 1 + tests/entrypoints/openai/test_serving_models.py | 1 + tests/entrypoints/openai/test_serving_responses.py | 2 ++ 4 files changed, 12 insertions(+) diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 9ea65f9fa6e7..706a69ea1c0e 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -401,6 +401,7 @@ class MockEngine: model_config: MockModelConfig = field(default_factory=MockModelConfig) input_processor: MagicMock = field(default_factory=MagicMock) io_processor: MagicMock = field(default_factory=MagicMock) + renderer: MagicMock = field(default_factory=MagicMock) async def _async_serving_chat_init(): @@ -431,6 +432,7 @@ async def test_serving_chat_returns_correct_model_name(): mock_engine.model_config = MockModelConfig() mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() + mock_engine.renderer = MagicMock() serving_chat = _build_serving_chat(mock_engine) messages = [{"role": "user", "content": "what is 1+1?"}] @@ -461,6 +463,7 @@ async def test_serving_chat_should_set_correct_max_tokens(): mock_engine.model_config = MockModelConfig() mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() + mock_engine.renderer = MagicMock() serving_chat = _build_serving_chat(mock_engine) @@ -494,6 +497,7 @@ async def test_serving_chat_should_set_correct_max_tokens(): mock_engine.model_config = mock_model_config mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() + mock_engine.renderer = MagicMock() # Initialize the serving chat serving_chat = _build_serving_chat(mock_engine) @@ -539,6 +543,7 @@ async def test_serving_chat_should_set_correct_max_tokens(): mock_engine.model_config = mock_model_config mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() + mock_engine.renderer = MagicMock() # Initialize the serving chat serving_chat = _build_serving_chat(mock_engine) @@ -585,6 +590,7 @@ async def test_serving_chat_could_load_correct_generation_config(): mock_engine.model_config = mock_model_config mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() + mock_engine.renderer = MagicMock() # Initialize the serving chat serving_chat = _build_serving_chat(mock_engine) @@ -631,6 +637,7 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type): mock_engine.model_config = mock_model_config mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() + mock_engine.renderer = MagicMock() serving_chat = _build_serving_chat(mock_engine) @@ -664,6 +671,7 @@ async def test_serving_chat_data_parallel_rank_extraction(): mock_engine.model_config = MockModelConfig() mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() + mock_engine.renderer = MagicMock() # Mock the generate method to return an async generator async def mock_generate(*args, **kwargs): diff --git a/tests/entrypoints/openai/test_serving_engine.py b/tests/entrypoints/openai/test_serving_engine.py index 192c7cafb749..05f315ed027c 100644 --- a/tests/entrypoints/openai/test_serving_engine.py +++ b/tests/entrypoints/openai/test_serving_engine.py @@ -25,6 +25,7 @@ def serving() -> OpenAIServing: models.model_config = model_config models.input_processor = Mock() models.io_processor = Mock() + models.renderer = Mock() serving = OpenAIServing( engine_client=engine_client, diff --git a/tests/entrypoints/openai/test_serving_models.py b/tests/entrypoints/openai/test_serving_models.py index b585835a0667..2c135cd1dea3 100644 --- a/tests/entrypoints/openai/test_serving_models.py +++ b/tests/entrypoints/openai/test_serving_models.py @@ -32,6 +32,7 @@ async def _async_serving_models_init() -> OpenAIServingModels: mock_engine_client.model_config = mock_model_config mock_engine_client.input_processor = MagicMock() mock_engine_client.io_processor = MagicMock() + mock_engine_client.renderer = MagicMock() serving_models = OpenAIServingModels( engine_client=mock_engine_client, diff --git a/tests/entrypoints/openai/test_serving_responses.py b/tests/entrypoints/openai/test_serving_responses.py index 7d03dccec30d..2264227fdb61 100644 --- a/tests/entrypoints/openai/test_serving_responses.py +++ b/tests/entrypoints/openai/test_serving_responses.py @@ -130,6 +130,7 @@ async def serving_responses_instance(self): engine_client.input_processor = MagicMock() engine_client.io_processor = MagicMock() + engine_client.renderer = MagicMock() models = MagicMock() @@ -216,6 +217,7 @@ async def serving_responses_instance(self): engine_client.input_processor = MagicMock() engine_client.io_processor = MagicMock() + engine_client.renderer = MagicMock() models = MagicMock() From 38ad20b50326951db32273bbbb1e735632765b3a Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 8 Dec 2025 17:32:47 +0000 Subject: [PATCH 44/65] Update tests Signed-off-by: DarkLight1337 --- .buildkite/test-amd.yaml | 6 ++- .buildkite/test-pipeline.yaml | 6 ++- tests/entrypoints/openai/test_serving_chat.py | 33 +++++++------- tests/renderers/__init__.py | 0 .../test_hf.py} | 4 +- .../test_mistral.py} | 43 +++++-------------- 6 files changed, 38 insertions(+), 54 deletions(-) create mode 100644 tests/renderers/__init__.py rename tests/{entrypoints/openai/test_chat_template.py => renderers/test_hf.py} (98%) rename tests/{entrypoints/openai/test_serving_engine.py => renderers/test_mistral.py} (50%) diff --git a/.buildkite/test-amd.yaml b/.buildkite/test-amd.yaml index 6950ad774edd..e93b1ef994ac 100644 --- a/.buildkite/test-amd.yaml +++ b/.buildkite/test-amd.yaml @@ -61,8 +61,8 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 15min - timeout_in_minutes: 20 +- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 20min + timeout_in_minutes: 30 mirror_hardwares: [amdexperimental, amdproduction, amdtentative] agent_pool: mi325_1 grade: Blocking @@ -71,6 +71,7 @@ steps: - tests/test_inputs.py - tests/test_outputs.py - tests/multimodal + - tests/renderers - tests/standalone_tests/lazy_imports.py - tests/tokenizers_ - tests/transformers_utils @@ -81,6 +82,7 @@ steps: - pytest -v -s test_inputs.py - pytest -v -s test_outputs.py - pytest -v -s -m 'cpu_test' multimodal + - pytest -v -s renderers - pytest -v -s tokenizers_ - pytest -v -s transformers_utils - pytest -v -s config diff --git a/.buildkite/test-pipeline.yaml b/.buildkite/test-pipeline.yaml index 0a99994e243a..fe11e0ca28b8 100644 --- a/.buildkite/test-pipeline.yaml +++ b/.buildkite/test-pipeline.yaml @@ -57,13 +57,14 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 15min - timeout_in_minutes: 20 +- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) # 20min + timeout_in_minutes: 30 source_file_dependencies: - vllm/ - tests/test_inputs.py - tests/test_outputs.py - tests/multimodal + - tests/renderers - tests/standalone_tests/lazy_imports.py - tests/tokenizers_ - tests/transformers_utils @@ -74,6 +75,7 @@ steps: - pytest -v -s test_inputs.py - pytest -v -s test_outputs.py - pytest -v -s -m 'cpu_test' multimodal + - pytest -v -s renderers - pytest -v -s tokenizers_ - pytest -v -s transformers_utils - pytest -v -s config diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 706a69ea1c0e..8df5c5d0462d 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -14,7 +14,8 @@ from vllm.entrypoints.openai.protocol import ChatCompletionRequest from vllm.entrypoints.openai.serving_chat import OpenAIServingChat from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels -from vllm.tokenizers import get_tokenizer +from vllm.renderers.hf import HfRenderer +from vllm.tokenizers.registry import tokenizer_args_from_config from vllm.v1.engine.async_llm import AsyncLLM from ...utils import RemoteOpenAIServer @@ -367,6 +368,15 @@ def get_diff_sampling_param(self): return self.diff_sampling_param or {} +def _build_renderer(model_config: MockModelConfig): + _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config) + + return HfRenderer( + model_config, + tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name}, + ) + + def _build_serving_chat(engine: AsyncLLM) -> OpenAIServingChat: models = OpenAIServingModels( engine_client=engine, @@ -427,12 +437,11 @@ def test_async_serving_chat_init(): @pytest.mark.asyncio async def test_serving_chat_returns_correct_model_name(): mock_engine = MagicMock(spec=AsyncLLM) - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME) mock_engine.errored = False mock_engine.model_config = MockModelConfig() mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - mock_engine.renderer = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) serving_chat = _build_serving_chat(mock_engine) messages = [{"role": "user", "content": "what is 1+1?"}] @@ -458,12 +467,11 @@ async def return_model_name(*args): @pytest.mark.asyncio async def test_serving_chat_should_set_correct_max_tokens(): mock_engine = MagicMock(spec=AsyncLLM) - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME) mock_engine.errored = False mock_engine.model_config = MockModelConfig() mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - mock_engine.renderer = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) serving_chat = _build_serving_chat(mock_engine) @@ -492,12 +500,11 @@ async def test_serving_chat_should_set_correct_max_tokens(): # Reinitialize the engine with new settings mock_engine = MagicMock(spec=AsyncLLM) - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME) mock_engine.errored = False mock_engine.model_config = mock_model_config mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - mock_engine.renderer = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) # Initialize the serving chat serving_chat = _build_serving_chat(mock_engine) @@ -538,12 +545,11 @@ async def test_serving_chat_should_set_correct_max_tokens(): # Reinitialize the engine with new settings mock_engine = MagicMock(spec=AsyncLLM) - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME) mock_engine.errored = False mock_engine.model_config = mock_model_config mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - mock_engine.renderer = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) # Initialize the serving chat serving_chat = _build_serving_chat(mock_engine) @@ -585,12 +591,11 @@ async def test_serving_chat_could_load_correct_generation_config(): } mock_engine = MagicMock(spec=AsyncLLM) - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME) mock_engine.errored = False mock_engine.model_config = mock_model_config mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - mock_engine.renderer = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) # Initialize the serving chat serving_chat = _build_serving_chat(mock_engine) @@ -632,12 +637,11 @@ async def test_serving_chat_did_set_correct_cache_salt(model_type): mock_model_config.hf_config.model_type = model_type mock_engine = MagicMock(spec=AsyncLLM) - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME) mock_engine.errored = False mock_engine.model_config = mock_model_config mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - mock_engine.renderer = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) serving_chat = _build_serving_chat(mock_engine) @@ -666,12 +670,11 @@ async def test_serving_chat_data_parallel_rank_extraction(): """Test that data_parallel_rank is properly extracted from header and passed to engine.""" mock_engine = MagicMock(spec=AsyncLLM) - mock_engine.get_tokenizer.return_value = get_tokenizer(MODEL_NAME) mock_engine.errored = False mock_engine.model_config = MockModelConfig() mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - mock_engine.renderer = MagicMock() + mock_engine.renderer = _build_renderer(mock_engine.model_config) # Mock the generate method to return an async generator async def mock_generate(*args, **kwargs): diff --git a/tests/renderers/__init__.py b/tests/renderers/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/tests/entrypoints/openai/test_chat_template.py b/tests/renderers/test_hf.py similarity index 98% rename from tests/entrypoints/openai/test_chat_template.py rename to tests/renderers/test_hf.py index 0762b4bcb696..fb9bd438a648 100644 --- a/tests/entrypoints/openai/test_chat_template.py +++ b/tests/renderers/test_hf.py @@ -9,8 +9,8 @@ from vllm.renderers.hf import safe_apply_chat_template from vllm.tokenizers import get_tokenizer -from ...models.registry import HF_EXAMPLE_MODELS -from ...utils import VLLM_PATH +from ..models.registry import HF_EXAMPLE_MODELS +from ..utils import VLLM_PATH chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja" assert chatml_jinja_path.exists() diff --git a/tests/entrypoints/openai/test_serving_engine.py b/tests/renderers/test_mistral.py similarity index 50% rename from tests/entrypoints/openai/test_serving_engine.py rename to tests/renderers/test_mistral.py index 05f315ed027c..40d846cb89b4 100644 --- a/tests/entrypoints/openai/test_serving_engine.py +++ b/tests/renderers/test_mistral.py @@ -8,37 +8,12 @@ import pytest from vllm.config import ModelConfig -from vllm.entrypoints.openai.serving_engine import OpenAIServing -from vllm.entrypoints.openai.serving_models import OpenAIServingModels +from vllm.renderers.mistral import MistralRenderer from vllm.tokenizers.mistral import MistralTokenizer -@pytest.fixture() -def serving() -> OpenAIServing: - """Create a minimal OpenAIServing instance for testing.""" - - # Create minimal mocks - engine_client = Mock() - model_config = Mock(spec=ModelConfig) - model_config.max_model_len = 32768 - models = Mock(spec=OpenAIServingModels) - models.model_config = model_config - models.input_processor = Mock() - models.io_processor = Mock() - models.renderer = Mock() - - serving = OpenAIServing( - engine_client=engine_client, - models=models, - request_logger=None, - ) - return serving - - @pytest.mark.asyncio -async def test_async_mistral_tokenizer_does_not_block_event_loop( - serving: OpenAIServing, -): +async def test_async_mistral_tokenizer_does_not_block_event_loop(): expected_tokens = [1, 2, 3] # Mock the blocking version to sleep @@ -47,11 +22,11 @@ def mocked_apply_chat_template(*_args, **_kwargs): return expected_tokens mock_tokenizer = Mock(spec=MistralTokenizer) - mock_tokenizer.apply_chat_template.side_effect = mocked_apply_chat_template + mock_tokenizer.apply_chat_template = mocked_apply_chat_template + mock_renderer = MistralRenderer(Mock(spec=ModelConfig), tokenizer_kwargs={}) + mock_renderer._tokenizer = mock_tokenizer - task = serving._apply_mistral_chat_template_async( - tokenizer=mock_tokenizer, messages=[], chat_template=None, tools=[] - ) + task = mock_renderer.render_messages_async([]) # Ensure the event loop is not blocked blocked_count = 0 @@ -67,6 +42,8 @@ def mocked_apply_chat_template(*_args, **_kwargs): await asyncio.sleep(0.1) # Ensure task completes - tokens = await task - assert tokens == expected_tokens, "Mocked blocking tokenizer was not called" + _, prompt = await task + assert prompt["prompt_token_ids"] == expected_tokens, ( + "Mocked blocking tokenizer was not called" + ) assert blocked_count == 0, "Event loop blocked during tokenization" From 89cbe7e29fadcdb0a4376c570ea76571bf5f9898 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Tue, 9 Dec 2025 04:26:45 +0000 Subject: [PATCH 45/65] Move more tests Signed-off-by: DarkLight1337 --- tests/entrypoints/test_chat_utils.py | 453 --------------------------- tests/renderers/test_hf.py | 381 +++++++++++++++++++++- tests/renderers/test_mistral.py | 53 +++- vllm/entrypoints/chat_utils.py | 2 + 4 files changed, 434 insertions(+), 455 deletions(-) diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index 99a18b20c7c6..16572f2b55d6 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -7,21 +7,14 @@ import pytest import torch -from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy from vllm.assets.audio import AudioAsset from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( - _try_extract_ast, - apply_mistral_chat_template, - load_chat_template, parse_chat_messages, parse_chat_messages_futures, - resolve_chat_template_content_format, - resolve_chat_template_kwargs, - resolve_hf_chat_template, ) from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict from vllm.multimodal.utils import ( @@ -29,25 +22,11 @@ encode_image_base64, encode_video_base64, ) -from vllm.tokenizers import get_tokenizer -from vllm.tokenizers.mistral import MistralTokenizer from vllm.utils.serial_utils import tensor2base64 -from ..models.registry import HF_EXAMPLE_MODELS -from ..utils import VLLM_PATH - -EXAMPLES_DIR = VLLM_PATH / "examples" - PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct" -ULTRAVOX_MODEL_ID = "fixie-ai/ultravox-v0_5-llama-3_2-1b" QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct" -QWEN2VL_MODEL_ID = "Qwen/Qwen2-VL-2B-Instruct" -QWEN25VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct" QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B" -QWEN3_MODEL_ID = "Qwen/Qwen3-8B" -LLAMA_GUARD_MODEL_ID = "meta-llama/Llama-Guard-3-1B" -HERMES_MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B" -MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" @pytest.fixture(scope="function") @@ -127,17 +106,6 @@ def qwen25omni_model_config_mm_interleaved(): ) -@pytest.fixture(scope="function") -def mistral_model_config(): - return ModelConfig( - MISTRAL_MODEL_ID, - runner="generate", - limit_mm_per_prompt={ - "image": 2, - }, - ) - - @pytest.fixture(scope="module") def image_url(): image = ImageAsset("cherry_blossom") @@ -1933,377 +1901,6 @@ def test_parse_chat_messages_multiple_images_interleave_with_placeholders( ) -@pytest.mark.parametrize( - "model", - [ - QWEN2VL_MODEL_ID, # tokenizer.chat_template is of type str - HERMES_MODEL_ID, # tokenizer.chat_template is of type dict - ], -) -@pytest.mark.parametrize("use_tools", [True, False]) -def test_resolve_hf_chat_template(sample_json_schema, model, use_tools): - """checks that chat_template is a dict type for HF models.""" - model_info = HF_EXAMPLE_MODELS.find_hf_info(model) - model_info.check_available_online(on_fail="skip") - - model_config = ModelConfig( - model, - tokenizer=model_info.tokenizer or model, - tokenizer_mode=model_info.tokenizer_mode, - revision=model_info.revision, - trust_remote_code=model_info.trust_remote_code, - hf_overrides=model_info.hf_overrides, - skip_tokenizer_init=model_info.require_embed_inputs, - enable_prompt_embeds=model_info.require_embed_inputs, - enable_mm_embeds=model_info.require_embed_inputs, - enforce_eager=model_info.enforce_eager, - dtype=model_info.dtype, - ) - - # Build the tokenizer - tokenizer = get_tokenizer( - model, - trust_remote_code=model_config.trust_remote_code, - ) - - tools = ( - [ - { - "type": "function", - "function": { - "name": "dummy_function_name", - "description": "This is a dummy function", - "parameters": sample_json_schema, - }, - } - ] - if use_tools - else None - ) - - # Test detecting the tokenizer's chat_template - chat_template = resolve_hf_chat_template( - tokenizer, - chat_template=None, - tools=tools, - model_config=model_config, - ) - assert isinstance(chat_template, str) - - -@pytest.mark.parametrize( - "model, expected_kwargs", - [ - ( - QWEN2VL_MODEL_ID, - { - "add_vision_id", - "add_generation_prompt", - "continue_final_message", - "tools", - }, - ), - ( - QWEN3_MODEL_ID, - { - "enable_thinking", - "add_generation_prompt", - "continue_final_message", - "tools", - }, - ), - ], -) -def test_resolve_hf_chat_template_kwargs(sample_json_schema, model, expected_kwargs): - """checks that chat_template is a dict type for HF models.""" - model_info = HF_EXAMPLE_MODELS.find_hf_info(model) - model_info.check_available_online(on_fail="skip") - - tools = [ - { - "type": "function", - "function": { - "name": "dummy_function_name", - "description": "This is a dummy function", - "parameters": sample_json_schema, - }, - } - ] - - chat_template_kwargs = { - # both unused - "unsed_kwargs_1": 123, - "unsed_kwargs_2": "abc", - # should not appear - "chat_template": "{% Hello world! %}", - "tokenize": True, - # used by tokenizer - "continue_final_message": True, - "tools": tools, - # both used by Qwen2-VL and Qwen3 - "add_generation_prompt": True, - # only used by Qwen2-VL - "add_vision_id": True, - # only used by Qwen3 - "enable_thinking": True, - } - - model_config = ModelConfig( - model, - tokenizer=model_info.tokenizer or model, - tokenizer_mode=model_info.tokenizer_mode, - revision=model_info.revision, - trust_remote_code=model_info.trust_remote_code, - hf_overrides=model_info.hf_overrides, - skip_tokenizer_init=model_info.require_embed_inputs, - enable_prompt_embeds=model_info.require_embed_inputs, - enable_mm_embeds=model_info.require_embed_inputs, - enforce_eager=model_info.enforce_eager, - dtype=model_info.dtype, - ) - - # Build the tokenizer - tokenizer = get_tokenizer( - model, - trust_remote_code=model_config.trust_remote_code, - ) - - # Test detecting the tokenizer's chat_template - chat_template = resolve_hf_chat_template( - tokenizer, - chat_template=None, - tools=tools, - model_config=model_config, - ) - with pytest.raises( - ValueError, match="Found unexpected chat template kwargs from request" - ): - # should raise error if `chat_template_kwargs` contains - # `chat_template` or `tokenize` - resolve_chat_template_kwargs( - tokenizer, - chat_template=chat_template, - chat_template_kwargs=chat_template_kwargs, - ) - resolved_chat_template_kwargs = resolve_chat_template_kwargs( - tokenizer, - chat_template=chat_template, - chat_template_kwargs=chat_template_kwargs, - raise_on_unexpected=False, - ) - assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs - - # Additional test: Verify HF base parameters work with **kwargs tokenizers - # This validates the fix for tokenizers like Kimi K2 that use **kwargs - # to receive standard HuggingFace parameters instead of declaring them explicitly - from vllm.entrypoints.chat_utils import _get_hf_base_chat_template_params - - hf_base_params = _get_hf_base_chat_template_params() - # Verify common HF parameters are in the base class - assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset( - hf_base_params - ), f"Expected HF base params not found in {hf_base_params}" - - # Test with a mock tokenizer that uses **kwargs (like Kimi K2) - class MockTokenizerWithKwargs: - def apply_chat_template(self, conversation, **kwargs): - return "mocked_output" - - mock_tokenizer = MockTokenizerWithKwargs() - mock_kwargs = { - "add_generation_prompt": True, - "tools": tools, - "continue_final_message": False, - "unknown_param": "should_be_filtered", - } - resolved_mock = resolve_chat_template_kwargs( - mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False - ) - # HF base params should pass through even with **kwargs tokenizer - assert "add_generation_prompt" in resolved_mock - assert "tools" in resolved_mock - assert "continue_final_message" in resolved_mock - # Unknown params should be filtered out - assert "unknown_param" not in resolved_mock - - -# NOTE: Qwen2-Audio default chat template is specially defined inside -# processor class instead of using `tokenizer_config.json` -@pytest.mark.parametrize( - ("model", "expected_format"), - [ - (PHI3V_MODEL_ID, "string"), - (QWEN2VL_MODEL_ID, "openai"), - (QWEN25VL_MODEL_ID, "openai"), - (ULTRAVOX_MODEL_ID, "string"), - (QWEN2AUDIO_MODEL_ID, "openai"), - (LLAMA_GUARD_MODEL_ID, "openai"), - ], -) -def test_resolve_content_format_hf_defined(model, expected_format): - model_info = HF_EXAMPLE_MODELS.find_hf_info(model) - model_info.check_available_online(on_fail="skip") - - model_config = ModelConfig( - model, - tokenizer=model_info.tokenizer or model, - tokenizer_mode=model_info.tokenizer_mode, - revision=model_info.revision, - trust_remote_code=model_info.trust_remote_code, - hf_overrides=model_info.hf_overrides, - skip_tokenizer_init=model_info.require_embed_inputs, - enable_prompt_embeds=model_info.require_embed_inputs, - enable_mm_embeds=model_info.require_embed_inputs, - enforce_eager=model_info.enforce_eager, - dtype=model_info.dtype, - ) - - tokenizer = get_tokenizer( - model, - trust_remote_code=model_config.trust_remote_code, - ) - - # Test detecting the tokenizer's chat_template - chat_template = resolve_hf_chat_template( - tokenizer, - chat_template=None, - tools=None, - model_config=model_config, - ) - assert isinstance(chat_template, str) - - print("[TEXT]") - print(chat_template) - print("[AST]") - print(_try_extract_ast(chat_template)) - - resolved_format = resolve_chat_template_content_format( - None, # Test detecting the tokenizer's chat_template - None, - "auto", - tokenizer, - model_config=model_config, - ) - - assert resolved_format == expected_format - - -@pytest.mark.parametrize( - ("model", "expected_format"), - [ - ("Salesforce/blip2-opt-2.7b", "string"), - ("facebook/chameleon-7b", "string"), - ("deepseek-ai/deepseek-vl2-tiny", "string"), - ("adept/fuyu-8b", "string"), - ("google/paligemma-3b-mix-224", "string"), - ("Qwen/Qwen-VL", "string"), - ("Qwen/Qwen-VL-Chat", "string"), - ], -) -def test_resolve_content_format_fallbacks(model, expected_format): - model_info = HF_EXAMPLE_MODELS.find_hf_info(model) - model_info.check_available_online(on_fail="skip") - - model_config = ModelConfig( - model, - tokenizer=model_info.tokenizer or model, - tokenizer_mode=model_info.tokenizer_mode, - revision=model_info.revision, - trust_remote_code=model_info.trust_remote_code, - hf_overrides=model_info.hf_overrides, - skip_tokenizer_init=model_info.require_embed_inputs, - enable_prompt_embeds=model_info.require_embed_inputs, - enable_mm_embeds=model_info.require_embed_inputs, - enforce_eager=model_info.enforce_eager, - dtype=model_info.dtype, - ) - - tokenizer = get_tokenizer( - model_config.tokenizer, - trust_remote_code=model_config.trust_remote_code, - ) - - # Test detecting the tokenizer's chat_template - chat_template = resolve_hf_chat_template( - tokenizer, - chat_template=None, - tools=None, - model_config=model_config, - ) - assert isinstance(chat_template, str) - - print("[TEXT]") - print(chat_template) - print("[AST]") - print(_try_extract_ast(chat_template)) - - resolved_format = resolve_chat_template_content_format( - None, # Test detecting the tokenizer's chat_template - None, - "auto", - tokenizer, - model_config=model_config, - ) - - assert resolved_format == expected_format - - -@pytest.mark.parametrize( - ("template_path", "expected_format"), - [ - ("template_alpaca.jinja", "string"), - ("template_baichuan.jinja", "string"), - ("template_chatglm.jinja", "string"), - ("template_chatglm2.jinja", "string"), - ("template_chatml.jinja", "string"), - ("template_dse_qwen2_vl.jinja", "openai"), - ("template_falcon_180b.jinja", "string"), - ("template_falcon.jinja", "string"), - ("template_inkbot.jinja", "string"), - ("template_teleflm.jinja", "string"), - ("template_vlm2vec_phi3v.jinja", "openai"), - ("template_vlm2vec_qwen2vl.jinja", "openai"), - ("tool_chat_template_granite_20b_fc.jinja", "string"), - ("tool_chat_template_hermes.jinja", "string"), - ("tool_chat_template_internlm2_tool.jinja", "string"), - ("tool_chat_template_llama3.1_json.jinja", "openai"), - ("tool_chat_template_llama3.2_json.jinja", "openai"), - ("tool_chat_template_mistral_parallel.jinja", "string"), - ("tool_chat_template_mistral.jinja", "string"), - ], -) -def test_resolve_content_format_examples(template_path, expected_format): - model_config = ModelConfig( - PHI3V_MODEL_ID, # Dummy - tokenizer=PHI3V_MODEL_ID, # Dummy - trust_remote_code=True, - ) - - dummy_tokenizer = get_tokenizer( - PHI3V_MODEL_ID, # Dummy - trust_remote_code=model_config.trust_remote_code, - ) - dummy_tokenizer.chat_template = None - - chat_template = load_chat_template(EXAMPLES_DIR / template_path) - assert isinstance(chat_template, str) - - print("[TEXT]") - print(chat_template) - print("[AST]") - print(_try_extract_ast(chat_template)) - - resolved_format = resolve_chat_template_content_format( - chat_template, - None, - "auto", - dummy_tokenizer, - model_config=model_config, - ) - - assert resolved_format == expected_format - - def test_parse_chat_messages_include_thinking_chunk(mistral_model_config): messages = [ { @@ -2365,56 +1962,6 @@ def test_parse_chat_messages_include_thinking_chunk(mistral_model_config): assert conversation_with_thinking == expected_conversation -def test_apply_mistral_chat_template_thinking_chunk(): - messages = [ - { - "role": "system", - "content": [ - {"type": "text", "text": "You are a helpful assistant."}, - { - "type": "thinking", - "closed": True, - "thinking": "Only return the answer when you are confident.", - }, - ], - }, - {"role": "user", "content": "What is 2+2?"}, - { - "role": "assistant", - "content": [ - {"type": "text", "text": "Let me think about it."}, - {"type": "thinking", "closed": True, "thinking": "2+2 = 4"}, - { - "type": "text", - "text": "The answer is 4.", - }, - ], - }, - {"role": "user", "content": "Thanks, what is 3+3?"}, - ] - mistral_tokenizer = MistralTokenizer.from_pretrained( - "mistralai/Magistral-Small-2509" - ) - - tokens_ids = apply_mistral_chat_template( - mistral_tokenizer, messages, chat_template=None, tools=None - ) - - string_tokens = mistral_tokenizer.mistral.decode( - tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP - ) - - expected_tokens = ( - r"[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the" - r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]" - r"[INST]What is 2+2?[/INST]" - r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4." - r"[INST]Thanks, what is 3+3?[/INST]" - ) - - assert string_tokens == expected_tokens - - def test_parse_chat_messages_single_empty_audio_with_uuid( qwen2_audio_model_config, ): diff --git a/tests/renderers/test_hf.py b/tests/renderers/test_hf.py index fb9bd438a648..168dfaa4b403 100644 --- a/tests/renderers/test_hf.py +++ b/tests/renderers/test_hf.py @@ -6,12 +6,21 @@ from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.openai.protocol import ChatCompletionRequest -from vllm.renderers.hf import safe_apply_chat_template +from vllm.renderers.hf import ( + _get_hf_base_chat_template_params, + _try_extract_ast, + resolve_chat_template, + resolve_chat_template_content_format, + resolve_chat_template_kwargs, + safe_apply_chat_template, +) from vllm.tokenizers import get_tokenizer from ..models.registry import HF_EXAMPLE_MODELS from ..utils import VLLM_PATH +EXAMPLES_DIR = VLLM_PATH / "examples" + chatml_jinja_path = VLLM_PATH / "examples/template_chatml.jinja" assert chatml_jinja_path.exists() @@ -98,6 +107,376 @@ def test_no_load_chat_template_literallike(): assert template_content == template +@pytest.mark.parametrize( + "model", + [ + "Qwen/Qwen2-VL-2B-Instruct", # chat_template is of type str + "NousResearch/Hermes-3-Llama-3.1-8B", # chat_template is of type dict + ], +) +@pytest.mark.parametrize("use_tools", [True, False]) +def test_resolve_chat_template(sample_json_schema, model, use_tools): + """checks that chat_template is a dict type for HF models.""" + model_info = HF_EXAMPLE_MODELS.find_hf_info(model) + model_info.check_available_online(on_fail="skip") + + model_config = ModelConfig( + model, + tokenizer=model_info.tokenizer or model, + tokenizer_mode=model_info.tokenizer_mode, + revision=model_info.revision, + trust_remote_code=model_info.trust_remote_code, + hf_overrides=model_info.hf_overrides, + skip_tokenizer_init=model_info.require_embed_inputs, + enable_prompt_embeds=model_info.require_embed_inputs, + enable_mm_embeds=model_info.require_embed_inputs, + enforce_eager=model_info.enforce_eager, + dtype=model_info.dtype, + ) + + # Build the tokenizer + tokenizer = get_tokenizer( + model, + trust_remote_code=model_config.trust_remote_code, + ) + + tools = ( + [ + { + "type": "function", + "function": { + "name": "dummy_function_name", + "description": "This is a dummy function", + "parameters": sample_json_schema, + }, + } + ] + if use_tools + else None + ) + + # Test detecting the tokenizer's chat_template + chat_template = resolve_chat_template( + tokenizer, + chat_template=None, + tools=tools, + model_config=model_config, + ) + assert isinstance(chat_template, str) + + +@pytest.mark.parametrize( + "model, expected_kwargs", + [ + ( + "Qwen/Qwen2-VL-2B-Instruct", + { + "add_vision_id", + "add_generation_prompt", + "continue_final_message", + "tools", + }, + ), + ( + "Qwen/Qwen3-8B", + { + "enable_thinking", + "add_generation_prompt", + "continue_final_message", + "tools", + }, + ), + ], +) +def test_resolve_chat_template_kwargs(sample_json_schema, model, expected_kwargs): + """checks that chat_template is a dict type for HF models.""" + model_info = HF_EXAMPLE_MODELS.find_hf_info(model) + model_info.check_available_online(on_fail="skip") + + tools = [ + { + "type": "function", + "function": { + "name": "dummy_function_name", + "description": "This is a dummy function", + "parameters": sample_json_schema, + }, + } + ] + + chat_template_kwargs = { + # both unused + "unsed_kwargs_1": 123, + "unsed_kwargs_2": "abc", + # should not appear + "chat_template": "{% Hello world! %}", + "tokenize": True, + # used by tokenizer + "continue_final_message": True, + "tools": tools, + # both used by Qwen2-VL and Qwen3 + "add_generation_prompt": True, + # only used by Qwen2-VL + "add_vision_id": True, + # only used by Qwen3 + "enable_thinking": True, + } + + model_config = ModelConfig( + model, + tokenizer=model_info.tokenizer or model, + tokenizer_mode=model_info.tokenizer_mode, + revision=model_info.revision, + trust_remote_code=model_info.trust_remote_code, + hf_overrides=model_info.hf_overrides, + skip_tokenizer_init=model_info.require_embed_inputs, + enable_prompt_embeds=model_info.require_embed_inputs, + enable_mm_embeds=model_info.require_embed_inputs, + enforce_eager=model_info.enforce_eager, + dtype=model_info.dtype, + ) + + # Build the tokenizer + tokenizer = get_tokenizer( + model, + trust_remote_code=model_config.trust_remote_code, + ) + + # Test detecting the tokenizer's chat_template + chat_template = resolve_chat_template( + tokenizer, + chat_template=None, + tools=tools, + model_config=model_config, + ) + with pytest.raises( + ValueError, match="Found unexpected chat template kwargs from request" + ): + # should raise error if `chat_template_kwargs` contains + # `chat_template` or `tokenize` + resolve_chat_template_kwargs( + tokenizer, + chat_template=chat_template, + chat_template_kwargs=chat_template_kwargs, + ) + resolved_chat_template_kwargs = resolve_chat_template_kwargs( + tokenizer, + chat_template=chat_template, + chat_template_kwargs=chat_template_kwargs, + raise_on_unexpected=False, + ) + assert set(resolved_chat_template_kwargs.keys()) == expected_kwargs + + # Additional test: Verify HF base parameters work with **kwargs tokenizers + # This validates the fix for tokenizers like Kimi K2 that use **kwargs + # to receive standard HuggingFace parameters instead of declaring them explicitly + hf_base_params = _get_hf_base_chat_template_params() + # Verify common HF parameters are in the base class + assert {"add_generation_prompt", "tools", "continue_final_message"}.issubset( + hf_base_params + ), f"Expected HF base params not found in {hf_base_params}" + + # Test with a mock tokenizer that uses **kwargs (like Kimi K2) + class MockTokenizerWithKwargs: + def apply_chat_template(self, conversation, **kwargs): + return "mocked_output" + + mock_tokenizer = MockTokenizerWithKwargs() + mock_kwargs = { + "add_generation_prompt": True, + "tools": tools, + "continue_final_message": False, + "unknown_param": "should_be_filtered", + } + resolved_mock = resolve_chat_template_kwargs( + mock_tokenizer, chat_template, mock_kwargs, raise_on_unexpected=False + ) + # HF base params should pass through even with **kwargs tokenizer + assert "add_generation_prompt" in resolved_mock + assert "tools" in resolved_mock + assert "continue_final_message" in resolved_mock + # Unknown params should be filtered out + assert "unknown_param" not in resolved_mock + + +# NOTE: Qwen2-Audio default chat template is specially defined inside +# processor class instead of using `tokenizer_config.json` +@pytest.mark.parametrize( + ("model", "expected_format"), + [ + ("microsoft/Phi-3.5-vision-instruct", "string"), + ("Qwen/Qwen2-VL-2B-Instruct", "openai"), + ("Qwen/Qwen2.5-VL-3B-Instruct", "openai"), + ("fixie-ai/ultravox-v0_5-llama-3_2-1b", "string"), + ("Qwen/Qwen2-Audio-7B-Instruct", "openai"), + ("meta-llama/Llama-Guard-3-1B", "openai"), + ], +) +def test_resolve_content_format_hf_defined(model, expected_format): + model_info = HF_EXAMPLE_MODELS.find_hf_info(model) + model_info.check_available_online(on_fail="skip") + + model_config = ModelConfig( + model, + tokenizer=model_info.tokenizer or model, + tokenizer_mode=model_info.tokenizer_mode, + revision=model_info.revision, + trust_remote_code=model_info.trust_remote_code, + hf_overrides=model_info.hf_overrides, + skip_tokenizer_init=model_info.require_embed_inputs, + enable_prompt_embeds=model_info.require_embed_inputs, + enable_mm_embeds=model_info.require_embed_inputs, + enforce_eager=model_info.enforce_eager, + dtype=model_info.dtype, + ) + + tokenizer = get_tokenizer( + model, + trust_remote_code=model_config.trust_remote_code, + ) + + # Test detecting the tokenizer's chat_template + chat_template = resolve_chat_template( + tokenizer, + chat_template=None, + tools=None, + model_config=model_config, + ) + assert isinstance(chat_template, str) + + print("[TEXT]") + print(chat_template) + print("[AST]") + print(_try_extract_ast(chat_template)) + + resolved_format = resolve_chat_template_content_format( + None, # Test detecting the tokenizer's chat_template + None, + "auto", + tokenizer, + model_config=model_config, + ) + + assert resolved_format == expected_format + + +@pytest.mark.parametrize( + ("model", "expected_format"), + [ + ("Salesforce/blip2-opt-2.7b", "string"), + ("facebook/chameleon-7b", "string"), + ("deepseek-ai/deepseek-vl2-tiny", "string"), + ("adept/fuyu-8b", "string"), + ("google/paligemma-3b-mix-224", "string"), + ("Qwen/Qwen-VL", "string"), + ("Qwen/Qwen-VL-Chat", "string"), + ], +) +def test_resolve_content_format_fallbacks(model, expected_format): + model_info = HF_EXAMPLE_MODELS.find_hf_info(model) + model_info.check_available_online(on_fail="skip") + + model_config = ModelConfig( + model, + tokenizer=model_info.tokenizer or model, + tokenizer_mode=model_info.tokenizer_mode, + revision=model_info.revision, + trust_remote_code=model_info.trust_remote_code, + hf_overrides=model_info.hf_overrides, + skip_tokenizer_init=model_info.require_embed_inputs, + enable_prompt_embeds=model_info.require_embed_inputs, + enable_mm_embeds=model_info.require_embed_inputs, + enforce_eager=model_info.enforce_eager, + dtype=model_info.dtype, + ) + + tokenizer = get_tokenizer( + model_config.tokenizer, + trust_remote_code=model_config.trust_remote_code, + ) + + # Test detecting the tokenizer's chat_template + chat_template = resolve_chat_template( + tokenizer, + chat_template=None, + tools=None, + model_config=model_config, + ) + assert isinstance(chat_template, str) + + print("[TEXT]") + print(chat_template) + print("[AST]") + print(_try_extract_ast(chat_template)) + + resolved_format = resolve_chat_template_content_format( + None, # Test detecting the tokenizer's chat_template + None, + "auto", + tokenizer, + model_config=model_config, + ) + + assert resolved_format == expected_format + + +@pytest.mark.parametrize( + ("template_path", "expected_format"), + [ + ("template_alpaca.jinja", "string"), + ("template_baichuan.jinja", "string"), + ("template_chatglm.jinja", "string"), + ("template_chatglm2.jinja", "string"), + ("template_chatml.jinja", "string"), + ("template_dse_qwen2_vl.jinja", "openai"), + ("template_falcon_180b.jinja", "string"), + ("template_falcon.jinja", "string"), + ("template_inkbot.jinja", "string"), + ("template_teleflm.jinja", "string"), + ("template_vlm2vec_phi3v.jinja", "openai"), + ("template_vlm2vec_qwen2vl.jinja", "openai"), + ("tool_chat_template_granite_20b_fc.jinja", "string"), + ("tool_chat_template_hermes.jinja", "string"), + ("tool_chat_template_internlm2_tool.jinja", "string"), + ("tool_chat_template_llama3.1_json.jinja", "openai"), + ("tool_chat_template_llama3.2_json.jinja", "openai"), + ("tool_chat_template_mistral_parallel.jinja", "string"), + ("tool_chat_template_mistral.jinja", "string"), + ], +) +def test_resolve_content_format_examples(template_path, expected_format): + model = "Qwen/Qwen2-VL-2B-Instruct" # Dummy + model_config = ModelConfig( + model, + tokenizer=model, + trust_remote_code=True, + ) + + dummy_tokenizer = get_tokenizer( + model, + trust_remote_code=model_config.trust_remote_code, + ) + dummy_tokenizer.chat_template = None + + chat_template = load_chat_template(EXAMPLES_DIR / template_path) + assert isinstance(chat_template, str) + + print("[TEXT]") + print(chat_template) + print("[AST]") + print(_try_extract_ast(chat_template)) + + resolved_format = resolve_chat_template_content_format( + chat_template, + None, + "auto", + dummy_tokenizer, + model_config=model_config, + ) + + assert resolved_format == expected_format + + @pytest.mark.parametrize( "model,template,add_generation_prompt,continue_final_message,expected_output", MODEL_TEMPLATE_GENERATION_OUTPUT, diff --git a/tests/renderers/test_mistral.py b/tests/renderers/test_mistral.py index 40d846cb89b4..0dc214ae939b 100644 --- a/tests/renderers/test_mistral.py +++ b/tests/renderers/test_mistral.py @@ -6,9 +6,10 @@ from unittest.mock import Mock import pytest +from mistral_common.tokens.tokenizers.base import SpecialTokenPolicy from vllm.config import ModelConfig -from vllm.renderers.mistral import MistralRenderer +from vllm.renderers.mistral import MistralRenderer, safe_apply_chat_template from vllm.tokenizers.mistral import MistralTokenizer @@ -47,3 +48,53 @@ def mocked_apply_chat_template(*_args, **_kwargs): "Mocked blocking tokenizer was not called" ) assert blocked_count == 0, "Event loop blocked during tokenization" + + +def test_apply_mistral_chat_template_thinking_chunk(): + messages = [ + { + "role": "system", + "content": [ + {"type": "text", "text": "You are a helpful assistant."}, + { + "type": "thinking", + "closed": True, + "thinking": "Only return the answer when you are confident.", + }, + ], + }, + {"role": "user", "content": "What is 2+2?"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": "Let me think about it."}, + {"type": "thinking", "closed": True, "thinking": "2+2 = 4"}, + { + "type": "text", + "text": "The answer is 4.", + }, + ], + }, + {"role": "user", "content": "Thanks, what is 3+3?"}, + ] + mistral_tokenizer = MistralTokenizer.from_pretrained( + "mistralai/Magistral-Small-2509" + ) + + tokens_ids = safe_apply_chat_template( + mistral_tokenizer, messages, chat_template=None, tools=None + ) + + string_tokens = mistral_tokenizer.mistral.decode( + tokens_ids, special_token_policy=SpecialTokenPolicy.KEEP + ) + + expected_tokens = ( + r"[SYSTEM_PROMPT]You are a helpful assistant.[THINK]Only return the" + r" answer when you are confident.[/THINK][/SYSTEM_PROMPT]" + r"[INST]What is 2+2?[/INST]" + r"Let me think about it.[THINK]2+2 = 4[/THINK]The answer is 4." + r"[INST]Thanks, what is 3+3?[/INST]" + ) + + assert string_tokens == expected_tokens diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 49b82bc521d1..c102c125a03d 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -60,6 +60,8 @@ def __getattr__(name: str): return resolve_chat_template + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + MODALITY_PLACEHOLDERS_MAP = { "image": "<##IMAGE##>", From 63d6961e835aa70bb66b5334a4e2760d4d1b14df Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Tue, 9 Dec 2025 04:31:32 +0000 Subject: [PATCH 46/65] Update misc Signed-off-by: DarkLight1337 --- .buildkite/test_areas/misc.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 072bccadb726..8d56809206fd 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -113,14 +113,15 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config (CPU) +- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) depends_on: ~ - timeout_in_minutes: 20 + timeout_in_minutes: 30 source_file_dependencies: - vllm/ - tests/test_inputs.py - tests/test_outputs.py - tests/multimodal + - tests/renderers - tests/standalone_tests/lazy_imports.py - tests/tokenizers_ - tests/transformers_utils @@ -131,6 +132,7 @@ steps: - pytest -v -s test_inputs.py - pytest -v -s test_outputs.py - pytest -v -s -m 'cpu_test' multimodal + - pytest -v -s renderers - pytest -v -s tokenizers_ - pytest -v -s transformers_utils - pytest -v -s config From 478b89e1bc7ba2ae87a8d49801aaf0672135e5ed Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Tue, 9 Dec 2025 14:20:14 +0000 Subject: [PATCH 47/65] Update Signed-off-by: DarkLight1337 --- .buildkite/test_areas/misc.yaml | 2 +- tests/entrypoints/test_chat_utils.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.buildkite/test_areas/misc.yaml b/.buildkite/test_areas/misc.yaml index 8d56809206fd..c6208b05ee59 100644 --- a/.buildkite/test_areas/misc.yaml +++ b/.buildkite/test_areas/misc.yaml @@ -113,7 +113,7 @@ steps: - pytest -v -s -m 'not cpu_test' multimodal - pytest -v -s utils_ -- label: Async Engine, Inputs, Utils, Worker, Config Test (CPU) +- label: Async Engine, Inputs, Utils, Worker, Config (CPU) depends_on: ~ timeout_in_minutes: 30 source_file_dependencies: diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index 16572f2b55d6..cd5975cea8db 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -27,6 +27,7 @@ PHI3V_MODEL_ID = "microsoft/Phi-3.5-vision-instruct" QWEN2AUDIO_MODEL_ID = "Qwen/Qwen2-Audio-7B-Instruct" QWEN25OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B" +MISTRAL_MODEL_ID = "mistralai/Mistral-Small-3.1-24B-Instruct-2503" @pytest.fixture(scope="function") @@ -106,6 +107,17 @@ def qwen25omni_model_config_mm_interleaved(): ) +@pytest.fixture(scope="function") +def mistral_model_config(): + return ModelConfig( + MISTRAL_MODEL_ID, + runner="generate", + limit_mm_per_prompt={ + "image": 2, + }, + ) + + @pytest.fixture(scope="module") def image_url(): image = ImageAsset("cherry_blossom") From 57283818716296b6468417f91884a1b107494f25 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 13 Dec 2025 08:10:33 +0000 Subject: [PATCH 48/65] Delay deprecation Signed-off-by: DarkLight1337 --- vllm/entrypoints/chat_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 1e0f0ec3c597..0c624332f776 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -60,7 +60,7 @@ def __getattr__(name: str): warnings.warn( "`vllm.entrypoints.chat_utils.resolve_hf_chat_template` has been moved to " "`vllm.renderers.hf.resolve_chat_template`. " - "The old name will be removed in v0.14.", + "The old name will be removed in v0.16.", DeprecationWarning, stacklevel=2, ) From 9cd0992b9a236105c1648342c007835ef28a4a50 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 14 Dec 2025 03:15:03 +0000 Subject: [PATCH 49/65] Fix test Signed-off-by: DarkLight1337 --- tests/entrypoints/openai/test_serving_chat.py | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 84a22adc5db1..9feaf7dc84bc 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -889,7 +889,7 @@ async def test_simple_chat(self, serving_chat, stream): # Test the Harmony messages for the first turn's input req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages, _, _ = serving_chat._make_request_with_harmony(req) + input_messages, _ = serving_chat._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -917,7 +917,7 @@ async def test_simple_chat(self, serving_chat, stream): # Test the Harmony messages for the second turn's input req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2) + input_messages_2, _ = serving_chat._make_request_with_harmony(req_2) verify_harmony_messages( input_messages_2, [ @@ -939,7 +939,7 @@ async def test_tool_call_response_with_content( # Test the Harmony messages for the first turn's input req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages, _, _ = serving_chat._make_request_with_harmony(req) + input_messages, _ = serving_chat._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -983,7 +983,7 @@ async def test_tool_call_response_with_content( # Test the Harmony messages for the second turn's input req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2) + input_messages_2, _ = serving_chat._make_request_with_harmony(req_2) verify_harmony_messages( input_messages_2, [ @@ -1020,7 +1020,7 @@ async def test_tools_and_reasoning( # Test the Harmony messages for the first turn's input req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages, _, _ = serving_chat._make_request_with_harmony(req) + input_messages, _ = serving_chat._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -1064,7 +1064,7 @@ async def test_tools_and_reasoning( # Test the Harmony messages for the second turn's input req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2) + input_messages_2, _ = serving_chat._make_request_with_harmony(req_2) verify_harmony_messages( input_messages_2, [ @@ -1101,7 +1101,7 @@ async def test_multi_turn_tools_and_reasoning( # Test the Harmony messages for the first turn's input req = ChatCompletionRequest(model=MODEL_NAME, messages=messages, tools=tools) - input_messages, _, _ = serving_chat._make_request_with_harmony(req) + input_messages, _ = serving_chat._make_request_with_harmony(req) verify_harmony_messages( input_messages, [ @@ -1145,7 +1145,7 @@ async def test_multi_turn_tools_and_reasoning( # Test the Harmony messages for the second turn's input req_2 = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages_2, _, _ = serving_chat._make_request_with_harmony(req_2) + input_messages_2, _ = serving_chat._make_request_with_harmony(req_2) verify_harmony_messages( input_messages_2, [ @@ -1195,7 +1195,7 @@ async def test_multi_turn_tools_and_reasoning( # Test the Harmony messages for the third turn's input req_3 = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages_3, _, _ = serving_chat._make_request_with_harmony(req_3) + input_messages_3, _ = serving_chat._make_request_with_harmony(req_3) verify_harmony_messages( input_messages_3, [ @@ -1258,7 +1258,7 @@ async def test_multi_turn_tools_and_reasoning( # Test the Harmony messages for the fourth turn's input req_4 = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages_4, _, _ = serving_chat._make_request_with_harmony(req_4) + input_messages_4, _ = serving_chat._make_request_with_harmony(req_4) verify_harmony_messages( input_messages_4, [ @@ -1307,7 +1307,7 @@ async def test_non_tool_reasoning(self, serving_chat): }, ] req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages, _, _ = serving_chat._make_request_with_harmony(req) + input_messages, _ = serving_chat._make_request_with_harmony(req) verify_harmony_messages( input_messages, @@ -1339,7 +1339,7 @@ async def test_non_tool_reasoning_empty_content(self, serving_chat): }, ] req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages, _, _ = serving_chat._make_request_with_harmony(req) + input_messages, _ = serving_chat._make_request_with_harmony(req) verify_harmony_messages( input_messages, @@ -1369,7 +1369,7 @@ async def test_non_tool_reasoning_empty_content_list(self, serving_chat): }, ] req = ChatCompletionRequest(model=MODEL_NAME, messages=messages) - input_messages, _, _ = serving_chat._make_request_with_harmony(req) + input_messages, _ = serving_chat._make_request_with_harmony(req) verify_harmony_messages( input_messages, From f332bc33d27160b71922d2d86ebd5e158a15f4c1 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 14 Dec 2025 09:33:23 +0000 Subject: [PATCH 50/65] Try rm Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 19cb7bc63e8c..568cb4c9509b 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -1156,7 +1156,7 @@ async def _preprocess_chat( tokenizer = renderer.get_tokenizer() request = tool_parser(tokenizer).adjust_request(request=request) # type: ignore - return conversation, [engine_prompt] # type: ignore[list-item] + return conversation, [engine_prompt] async def _process_inputs( self, From 01e419dce5835392a8f055225a77ea2edc1e4d98 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 14 Dec 2025 09:40:38 +0000 Subject: [PATCH 51/65] fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/serving_engine.py b/vllm/entrypoints/openai/serving_engine.py index 568cb4c9509b..49b0f5567bc0 100644 --- a/vllm/entrypoints/openai/serving_engine.py +++ b/vllm/entrypoints/openai/serving_engine.py @@ -8,7 +8,7 @@ from collections.abc import AsyncGenerator, Callable, Iterable, Mapping from dataclasses import dataclass, field from http import HTTPStatus -from typing import Any, ClassVar, Generic, TypeAlias, TypeVar +from typing import Any, ClassVar, Generic, TypeAlias, TypeVar, cast import numpy as np from fastapi import Request @@ -1133,6 +1133,8 @@ async def _preprocess_chat( # Fill in other keys like MM data engine_prompt.update(extra_data) # type: ignore + engine_prompt = cast(TokensPrompt, engine_prompt) + if request.mm_processor_kwargs is not None: engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs if (cache_salt := getattr(request, "cache_salt", None)) is not None: From 59b8e09a5b9e23c66623b82442945723e2420ef7 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 4 Jan 2026 08:37:06 +0000 Subject: [PATCH 52/65] Fix Signed-off-by: DarkLight1337 --- tests/v1/engine/test_process_multi_modal_uuids.py | 1 + vllm/entrypoints/openai/serving_chat.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/v1/engine/test_process_multi_modal_uuids.py b/tests/v1/engine/test_process_multi_modal_uuids.py index b83f888d6f11..8d6f00788786 100644 --- a/tests/v1/engine/test_process_multi_modal_uuids.py +++ b/tests/v1/engine/test_process_multi_modal_uuids.py @@ -12,6 +12,7 @@ MultiModalConfig, VllmConfig, ) +from vllm.multimodal.inputs import MultiModalUUIDDict from vllm.sampling_params import SamplingParams from vllm.v1.engine import input_processor as input_processor_mod from vllm.v1.engine.input_processor import InputProcessor diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index 72154fa0fdbe..eee04a0f7f90 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -179,8 +179,8 @@ async def warmup(self) -> None: start_time = time.perf_counter() try: - # Get the tokenizer from the engine - tokenizer = await self.engine_client.get_tokenizer() + renderer = self.engine_client.renderer + tokenizer = renderer.tokenizer # Create a minimal dummy request dummy_request = ChatCompletionRequest( From e34751f6b7eb041ae454dc053f3c7231e0371a0b Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 4 Jan 2026 09:03:49 +0000 Subject: [PATCH 53/65] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/serving_chat.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py index eee04a0f7f90..f7ec7cb623d0 100644 --- a/vllm/entrypoints/openai/serving_chat.py +++ b/vllm/entrypoints/openai/serving_chat.py @@ -180,7 +180,6 @@ async def warmup(self) -> None: try: renderer = self.engine_client.renderer - tokenizer = renderer.tokenizer # Create a minimal dummy request dummy_request = ChatCompletionRequest( @@ -196,7 +195,7 @@ async def warmup(self) -> None: # 3. Tokenizer initialization for chat await self._preprocess_chat( dummy_request, - tokenizer, + renderer, dummy_request.messages, chat_template=self.chat_template, chat_template_content_format=self.chat_template_content_format, From d24fe7b714a9a826ac961b68360f0205ff456d48 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 4 Jan 2026 09:15:05 +0000 Subject: [PATCH 54/65] Fix Signed-off-by: DarkLight1337 --- tests/entrypoints/pooling/score/test_utils.py | 8 ++++---- vllm/entrypoints/score_utils.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/entrypoints/pooling/score/test_utils.py b/tests/entrypoints/pooling/score/test_utils.py index 356fd0ad6678..7022c2560333 100644 --- a/tests/entrypoints/pooling/score/test_utils.py +++ b/tests/entrypoints/pooling/score/test_utils.py @@ -212,7 +212,7 @@ def test_not_using_default_template( return_value=mock_model_no_score_template, ), patch( - "vllm.entrypoints.score_utils.apply_hf_chat_template", + "vllm.entrypoints.score_utils.safe_apply_chat_template", return_value="test querytest doc", ), ): @@ -245,7 +245,7 @@ def test_fallback_with_pad_token( return_value=mock_model_no_score_template, ), patch( - "vllm.entrypoints.score_utils.apply_hf_chat_template", + "vllm.entrypoints.score_utils.safe_apply_chat_template", side_effect=ChatTemplateResolutionError("No template"), ), ): @@ -296,7 +296,7 @@ def test_fallback_without_pad_token( return_value=mock_model_no_score_template, ), patch( - "vllm.entrypoints.score_utils.apply_hf_chat_template", + "vllm.entrypoints.score_utils.safe_apply_chat_template", side_effect=ChatTemplateResolutionError("No template"), ), ): @@ -331,7 +331,7 @@ def test_post_process_tokens_called( return_value=mock_model_with_score_template, ), patch( - "vllm.entrypoints.score_utils.apply_hf_chat_template", + "vllm.entrypoints.score_utils.safe_apply_chat_template", side_effect=ChatTemplateResolutionError("No template"), ), ): diff --git a/vllm/entrypoints/score_utils.py b/vllm/entrypoints/score_utils.py index ba10a72fe7e0..6d9ce68d93f1 100644 --- a/vllm/entrypoints/score_utils.py +++ b/vllm/entrypoints/score_utils.py @@ -15,12 +15,12 @@ MultiModalItemTracker, _ContentPart, _parse_chat_message_content_part, - apply_hf_chat_template, ) from vllm.inputs import TokensPrompt from vllm.model_executor.models.interfaces import supports_score_template from vllm.multimodal.inputs import MultiModalDataDict from vllm.outputs import PoolingRequestOutput +from vllm.renderers.hf import safe_apply_chat_template from vllm.tokenizers import TokenizerLike ScoreContentPartParam: TypeAlias = ( @@ -222,7 +222,7 @@ def default_tokenizer_encode(): # If that fails because there is no such template, # fall back to the default implementation. try: - full_prompt = apply_hf_chat_template( + full_prompt = safe_apply_chat_template( tokenizer, [ {"role": "query", "content": prompt_1}, From eae64ca3589a9051d00785fbaa270ea5104b98b4 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 4 Jan 2026 09:17:47 +0000 Subject: [PATCH 55/65] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/score_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/entrypoints/score_utils.py b/vllm/entrypoints/score_utils.py index 6d9ce68d93f1..67a10c801a25 100644 --- a/vllm/entrypoints/score_utils.py +++ b/vllm/entrypoints/score_utils.py @@ -223,14 +223,14 @@ def default_tokenizer_encode(): # fall back to the default implementation. try: full_prompt = safe_apply_chat_template( + model_config, tokenizer, [ {"role": "query", "content": prompt_1}, {"role": "document", "content": prompt_2}, ], - score_template, + chat_template=score_template, tools=None, - model_config=model_config, ) prompt_inputs = tokenizer(full_prompt, **tokenization_kwargs) except ChatTemplateResolutionError: From fe4aae4de62581950e4ebae729cd0ddb7dc90fd8 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sun, 4 Jan 2026 09:20:20 +0000 Subject: [PATCH 56/65] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/score_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vllm/entrypoints/score_utils.py b/vllm/entrypoints/score_utils.py index 67a10c801a25..d78e1cf3aa1f 100644 --- a/vllm/entrypoints/score_utils.py +++ b/vllm/entrypoints/score_utils.py @@ -231,6 +231,7 @@ def default_tokenizer_encode(): ], chat_template=score_template, tools=None, + tokenize=False, ) prompt_inputs = tokenizer(full_prompt, **tokenization_kwargs) except ChatTemplateResolutionError: From ce521f97dee51a20c8152b76b6450cf16c781628 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Tue, 6 Jan 2026 06:15:06 +0000 Subject: [PATCH 57/65] Use new name Signed-off-by: DarkLight1337 --- vllm/renderers/{deepseekv32.py => deepseek_v32.py} | 2 +- vllm/renderers/registry.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename vllm/renderers/{deepseekv32.py => deepseek_v32.py} (98%) diff --git a/vllm/renderers/deepseekv32.py b/vllm/renderers/deepseek_v32.py similarity index 98% rename from vllm/renderers/deepseekv32.py rename to vllm/renderers/deepseek_v32.py index 6f7b7898c024..d0759db7a999 100644 --- a/vllm/renderers/deepseekv32.py +++ b/vllm/renderers/deepseek_v32.py @@ -12,7 +12,7 @@ from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger from vllm.tokenizers import cached_get_tokenizer -from vllm.tokenizers.deepseekv32 import DeepseekV32Tokenizer +from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer from .protocol import RendererLike diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index a6c402edf32a..929c83784f73 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -16,7 +16,7 @@ _VLLM_RENDERERS = { - "deepseekv32": ("deepseekv32", "DeepseekV32Renderer"), + "deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"), "hf": ("hf", "HfRenderer"), "mistral": ("mistral", "MistralRenderer"), "terratorch": ("terratorch", "TerratorchRenderer"), From dbd9bbb4eb76bc794115eb46fa47fd7b920f0494 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 9 Jan 2026 11:12:53 +0000 Subject: [PATCH 58/65] Address comment Signed-off-by: DarkLight1337 --- tests/entrypoints/test_chat_utils.py | 54 ++++++++++++++-------------- vllm/entrypoints/chat_utils.py | 6 ++-- vllm/renderers/deepseek_v32.py | 8 ++--- vllm/renderers/hf.py | 8 ++--- vllm/renderers/mistral.py | 8 ++--- vllm/renderers/terratorch.py | 8 ++--- 6 files changed, 46 insertions(+), 46 deletions(-) diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index f18dfb8abf1b..49237abfbd58 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -14,7 +14,7 @@ from vllm.config import ModelConfig from vllm.entrypoints.chat_utils import ( parse_chat_messages, - parse_chat_messages_futures, + parse_chat_messages_async, ) from vllm.multimodal import MultiModalDataDict, MultiModalUUIDDict from vllm.multimodal.utils import ( @@ -449,7 +449,7 @@ async def test_parse_chat_messages_single_image_with_uuid_async( image_url, ): image_uuid = str(hash(image_url)) - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -470,7 +470,7 @@ async def test_parse_chat_messages_single_image_with_uuid_async( assert conversation == [ {"role": "user", "content": "<|image_1|>\nWhat's in the image?"} ] - _assert_mm_data_is_image_input(await mm_future, 1) + _assert_mm_data_is_image_input(mm_data, 1) _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid]) @@ -480,7 +480,7 @@ async def test_parse_chat_messages_empty_image_with_uuid_async( image_url, ): image_uuid = str(hash(image_url)) - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -501,7 +501,7 @@ async def test_parse_chat_messages_empty_image_with_uuid_async( assert conversation == [ {"role": "user", "content": "<|image_1|>\nWhat's in the image?"} ] - _assert_mm_data_is_image_input(await mm_future, 1, skipped_image_indices=[0]) + _assert_mm_data_is_image_input(mm_data, 1, skipped_image_indices=[0]) _assert_mm_uuids(mm_uuids, 1, expected_uuids=[image_uuid]) @@ -513,7 +513,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async( image_uuid1 = "my_uuid_1" image_uuid2 = "my_uuid_2" - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -542,7 +542,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_async( "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?", } ] - _assert_mm_data_is_image_input(await mm_future, 2) + _assert_mm_data_is_image_input(mm_data, 2) _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2]) @@ -554,7 +554,7 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async( image_uuid1 = "my_uuid_1" image_uuid2 = "my_uuid_2" - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -583,7 +583,7 @@ async def test_parse_chat_messages_multiple_empty_images_with_uuids_async( "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?", } ] - _assert_mm_data_is_image_input(await mm_future, 2, skipped_image_indices=[0, 1]) + _assert_mm_data_is_image_input(mm_data, 2, skipped_image_indices=[0, 1]) _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid1, image_uuid2]) @@ -594,7 +594,7 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async( ): image_uuid2 = "my_uuid_2" - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -622,7 +622,7 @@ async def test_parse_chat_messages_multiple_images_with_partial_uuids_async( "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?", } ] - _assert_mm_data_is_image_input(await mm_future, 2) + _assert_mm_data_is_image_input(mm_data, 2) _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, image_uuid2]) @@ -669,7 +669,7 @@ async def test_parse_chat_messages_single_image_async( phi3v_model_config, image_url, ): - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -686,7 +686,7 @@ async def test_parse_chat_messages_single_image_async( assert conversation == [ {"role": "user", "content": "<|image_1|>\nWhat's in the image?"} ] - _assert_mm_data_is_image_input(await mm_future, 1) + _assert_mm_data_is_image_input(mm_data, 1) _assert_mm_uuids(mm_uuids, 1, expected_uuids=[None]) @@ -870,7 +870,7 @@ async def test_parse_chat_messages_audio_embeds_async( # Encode it as base64 base64_audio_embedding = tensor2base64(audio_embedding) - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -888,7 +888,7 @@ async def test_parse_chat_messages_audio_embeds_async( ) # Should have audio embedding in mm_data (single tensor, not a list) - mm_data = await mm_future + mm_data = mm_data assert mm_data is not None assert "audio" in mm_data assert isinstance(mm_data["audio"], torch.Tensor) @@ -1030,7 +1030,7 @@ async def test_parse_chat_messages_multiple_image_embeds_async( base64_image_embedding_1 = tensor2base64(image_embedding_1) base64_image_embedding_2 = tensor2base64(image_embedding_2) - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -1060,7 +1060,7 @@ async def test_parse_chat_messages_multiple_image_embeds_async( ] # Await the future and verify mm_data - mm_data = await mm_future + mm_data = mm_data assert mm_data is not None assert "image" in mm_data assert isinstance(mm_data["image"], list) @@ -1081,7 +1081,7 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async( phi3v_model_config_image_embeds, ): uuid = "abcd" - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -1101,7 +1101,7 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async( "content": "<|image_1|>\nWhat's in this image?", } ] - mm_data = await mm_future + mm_data = mm_data assert mm_data is not None assert "image" in mm_data assert isinstance(mm_data["image"], list) @@ -1208,7 +1208,7 @@ async def test_parse_chat_messages_multiple_images_async( phi3v_model_config, image_url, ): - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -1232,7 +1232,7 @@ async def test_parse_chat_messages_multiple_images_async( "content": "<|image_1|>\n<|image_2|>\nWhat's in these images?", } ] - _assert_mm_data_is_image_input(await mm_future, 2) + _assert_mm_data_is_image_input(mm_data, 2) _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None]) @@ -1562,7 +1562,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async( phi3v_model_config_mm_interleaved, image_url, ): - conversation, mm_data, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -1589,7 +1589,7 @@ async def test_parse_chat_messages_multiple_images_interleave_async( "Do they have differences?", } ] - _assert_mm_data_is_image_input(await mm_data, 2) + _assert_mm_data_is_image_input(mm_data, 2) _assert_mm_uuids(mm_uuids, 2, expected_uuids=[None, None]) @@ -1599,7 +1599,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async( image_url, ): image_uuid = str(hash(image_url)) - conversation, mm_data, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -1634,7 +1634,7 @@ async def test_parse_chat_messages_multiple_images_with_uuids_interleave_async( "Do they have differences?", } ] - _assert_mm_data_is_image_input(await mm_data, 2) + _assert_mm_data_is_image_input(mm_data, 2) _assert_mm_uuids(mm_uuids, 2, expected_uuids=[image_uuid, image_uuid]) @@ -2109,7 +2109,7 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async( qwen2_audio_model_config, ): audio_uuid = "abcd" - conversation, mm_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( [ { "role": "user", @@ -2134,5 +2134,5 @@ async def test_parse_chat_messages_single_empty_audio_with_uuid_async( "audio say?", } ] - _assert_mm_data_inputs(await mm_future, {"audio": 1}) + _assert_mm_data_inputs(mm_data, {"audio": 1}) _assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[audio_uuid]) diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 5db15451ae9a..eb796c9661f9 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -1383,13 +1383,13 @@ def parse_chat_messages( return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids() -def parse_chat_messages_futures( +async def parse_chat_messages_async( messages: list[ChatCompletionMessageParam], model_config: ModelConfig, content_format: ChatTemplateContentFormat, ) -> tuple[ list[ConversationMessage], - Awaitable[MultiModalDataDict | None], + MultiModalDataDict | None, MultiModalUUIDDict | None, ]: conversation: list[ConversationMessage] = [] @@ -1411,7 +1411,7 @@ def parse_chat_messages_futures( _postprocess_messages(conversation) - return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids() + return conversation, await mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids() def get_history_tool_calls_cnt(conversation: list[ConversationMessage]): diff --git a/vllm/renderers/deepseek_v32.py b/vllm/renderers/deepseek_v32.py index d0759db7a999..123911654d8c 100644 --- a/vllm/renderers/deepseek_v32.py +++ b/vllm/renderers/deepseek_v32.py @@ -7,7 +7,7 @@ ChatCompletionMessageParam, ConversationMessage, parse_chat_messages, - parse_chat_messages_futures, + parse_chat_messages_async, ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger @@ -94,7 +94,7 @@ async def render_messages_async( **kwargs, ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: tokenizer = self.get_tokenizer() - conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( messages, self.config, content_format="string", @@ -111,8 +111,8 @@ async def render_messages_async( if isinstance(prompt_raw, str) else TokensPrompt(prompt_token_ids=prompt_raw) ) - if mm_data_future is not None: - prompt["multi_modal_data"] = await mm_data_future + if mm_data is not None: + prompt["multi_modal_data"] = mm_data if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 6cbd2a110752..d2252c65544b 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -22,7 +22,7 @@ ConversationMessage, load_chat_template, parse_chat_messages, - parse_chat_messages_futures, + parse_chat_messages_async, ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger @@ -568,7 +568,7 @@ async def render_messages_async( model_config = self.config tokenizer = self.get_tokenizer() - conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( messages, model_config, content_format=resolve_chat_template_content_format( @@ -592,8 +592,8 @@ async def render_messages_async( if isinstance(prompt_raw, str) else TokensPrompt(prompt_token_ids=prompt_raw) ) - if mm_data_future is not None: - prompt["multi_modal_data"] = await mm_data_future + if mm_data is not None: + prompt["multi_modal_data"] = mm_data if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index 9e57fffb0800..c45fb1f77ed8 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -8,7 +8,7 @@ ChatCompletionMessageParam, ConversationMessage, parse_chat_messages, - parse_chat_messages_futures, + parse_chat_messages_async, ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger @@ -124,7 +124,7 @@ async def render_messages_async( **kwargs, ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: tokenizer = self.get_tokenizer() - conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( messages, self.config, content_format="string", @@ -139,8 +139,8 @@ async def render_messages_async( if isinstance(prompt_raw, str) else TokensPrompt(prompt_token_ids=prompt_raw) ) - if mm_data_future is not None: - prompt["multi_modal_data"] = await mm_data_future + if mm_data is not None: + prompt["multi_modal_data"] = mm_data if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids diff --git a/vllm/renderers/terratorch.py b/vllm/renderers/terratorch.py index 6a3f28c7aa07..fc41a94c85b2 100644 --- a/vllm/renderers/terratorch.py +++ b/vllm/renderers/terratorch.py @@ -7,7 +7,7 @@ ChatCompletionMessageParam, ConversationMessage, parse_chat_messages, - parse_chat_messages_futures, + parse_chat_messages_async, ) from vllm.inputs import TextPrompt, TokensPrompt from vllm.logger import init_logger @@ -70,15 +70,15 @@ async def render_messages_async( ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: model_config = self.config - conversation, mm_data_future, mm_uuids = parse_chat_messages_futures( + conversation, mm_data, mm_uuids = await parse_chat_messages_async( messages, model_config, content_format="string", ) prompt = TokensPrompt(prompt_token_ids=[1]) # Dummy token IDs - if mm_data_future is not None: - prompt["multi_modal_data"] = await mm_data_future + if mm_data is not None: + prompt["multi_modal_data"] = mm_data if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids From 4899b9edb6df324e06005ac86d95b1b7abfc65e1 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 9 Jan 2026 11:14:20 +0000 Subject: [PATCH 59/65] Handle grok2 Signed-off-by: DarkLight1337 --- vllm/renderers/grok2.py | 119 +++++++++++++++++++++++++++++++++++++ vllm/renderers/registry.py | 1 + 2 files changed, 120 insertions(+) create mode 100644 vllm/renderers/grok2.py diff --git a/vllm/renderers/grok2.py b/vllm/renderers/grok2.py new file mode 100644 index 000000000000..06de760f8f90 --- /dev/null +++ b/vllm/renderers/grok2.py @@ -0,0 +1,119 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from typing import Any + +from vllm.config import ModelConfig +from vllm.entrypoints.chat_utils import ( + ChatCompletionMessageParam, + ConversationMessage, + parse_chat_messages, + parse_chat_messages_async, +) +from vllm.inputs import TextPrompt, TokensPrompt +from vllm.logger import init_logger +from vllm.tokenizers import cached_get_tokenizer +from vllm.tokenizers.grok2 import Grok2Tokenizer + +from .protocol import RendererLike + +logger = init_logger(__name__) + + +class Grok2Renderer(RendererLike): + @classmethod + def from_config( + cls, + config: ModelConfig, + tokenizer_kwargs: dict[str, Any], + ) -> "RendererLike": + return cls(config, tokenizer_kwargs) + + def __init__( + self, + config: ModelConfig, + tokenizer_kwargs: dict[str, Any], + ) -> None: + super().__init__() + + self.config = config + + if config.skip_tokenizer_init: + tokenizer = None + else: + tokenizer = cached_get_tokenizer( + tokenizer_cls=Grok2Tokenizer, + **tokenizer_kwargs, + ) + + self._tokenizer = tokenizer + + @property + def tokenizer(self) -> Grok2Tokenizer | None: + return self._tokenizer + + def get_tokenizer(self) -> Grok2Tokenizer: + tokenizer = self.tokenizer + if tokenizer is None: + raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") + + return tokenizer + + def render_messages( + self, + messages: list[ChatCompletionMessageParam], + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + tokenizer = self.get_tokenizer() + conversation, mm_data, mm_uuids = parse_chat_messages( + messages, + self.config, + content_format="string", + ) + + prompt_raw = tokenizer.apply_chat_template( + conversation=conversation, + messages=messages, + **kwargs, + ) + + prompt = ( + TextPrompt(prompt=prompt_raw) + if isinstance(prompt_raw, str) + else TokensPrompt(prompt_token_ids=prompt_raw) + ) + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt # type: ignore[return-value] + + async def render_messages_async( + self, + messages: list[ChatCompletionMessageParam], + **kwargs, + ) -> tuple[list[ConversationMessage], TextPrompt | TokensPrompt]: + tokenizer = self.get_tokenizer() + conversation, mm_data, mm_uuids = await parse_chat_messages_async( + messages, + self.config, + content_format="string", + ) + + prompt_raw = tokenizer.apply_chat_template( + conversation=conversation, + messages=messages, + **kwargs, + ) + + prompt = ( + TextPrompt(prompt=prompt_raw) + if isinstance(prompt_raw, str) + else TokensPrompt(prompt_token_ids=prompt_raw) + ) + if mm_data is not None: + prompt["multi_modal_data"] = mm_data + if mm_uuids is not None: + prompt["multi_modal_uuids"] = mm_uuids + + return conversation, prompt # type: ignore[return-value] diff --git a/vllm/renderers/registry.py b/vllm/renderers/registry.py index 929c83784f73..5269978b5b2f 100644 --- a/vllm/renderers/registry.py +++ b/vllm/renderers/registry.py @@ -18,6 +18,7 @@ _VLLM_RENDERERS = { "deepseek_v32": ("deepseek_v32", "DeepseekV32Renderer"), "hf": ("hf", "HfRenderer"), + "grok2": ("grok2", "Grok2Renderer"), "mistral": ("mistral", "MistralRenderer"), "terratorch": ("terratorch", "TerratorchRenderer"), } From d431a119aba184c06baf47ec071ef7388866136d Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 9 Jan 2026 12:36:22 +0000 Subject: [PATCH 60/65] Address comment Signed-off-by: DarkLight1337 --- vllm/entrypoints/llm.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index acf00a90dfa7..540551f8b18e 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -70,6 +70,7 @@ from vllm.sampling_params import BeamSearchParams, RequestOutputKind, SamplingParams from vllm.tasks import PoolingTask from vllm.tokenizers import TokenizerLike +from vllm.tokenizers.mistral import MistralTokenizer from vllm.usage.usage_lib import UsageContext from vllm.utils.collection_utils import as_iter, is_list_of from vllm.utils.counter import Counter @@ -1246,6 +1247,9 @@ def _cross_encoding_score( ) -> list[ScoringRequestOutput]: model_config = self.model_config + if isinstance(tokenizer, MistralTokenizer): + raise ValueError("Score API is not supported for Mistral tokenizer") + if len(data_1) == 1: data_1 = data_1 * len(data_2) From 6e0b29142a21449b4c217f38de18e738c8592f19 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 9 Jan 2026 14:14:14 +0000 Subject: [PATCH 61/65] Remove redundant statements Signed-off-by: DarkLight1337 --- tests/entrypoints/test_chat_utils.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/entrypoints/test_chat_utils.py b/tests/entrypoints/test_chat_utils.py index 49237abfbd58..ba43f37bd27b 100644 --- a/tests/entrypoints/test_chat_utils.py +++ b/tests/entrypoints/test_chat_utils.py @@ -888,7 +888,6 @@ async def test_parse_chat_messages_audio_embeds_async( ) # Should have audio embedding in mm_data (single tensor, not a list) - mm_data = mm_data assert mm_data is not None assert "audio" in mm_data assert isinstance(mm_data["audio"], torch.Tensor) @@ -1060,7 +1059,6 @@ async def test_parse_chat_messages_multiple_image_embeds_async( ] # Await the future and verify mm_data - mm_data = mm_data assert mm_data is not None assert "image" in mm_data assert isinstance(mm_data["image"], list) @@ -1101,7 +1099,6 @@ async def test_parse_chat_messages_empty_image_embeds_with_uuid_async( "content": "<|image_1|>\nWhat's in this image?", } ] - mm_data = mm_data assert mm_data is not None assert "image" in mm_data assert isinstance(mm_data["image"], list) From b77888bdd3abbcc2d8a5caa63b9031b95104c3d2 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Thu, 15 Jan 2026 09:09:23 +0000 Subject: [PATCH 62/65] Address comment Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/engine/serving.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py index 73bd71dea3dd..5fc97fe91fd2 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/openai/engine/serving.py @@ -1196,7 +1196,10 @@ async def _preprocess_chat( conversation, engine_prompt = await renderer.render_messages_async( messages, chat_template_content_format=chat_template_content_format, - tokenize=isinstance(renderer.tokenizer, MistralTokenizer), + tokenize=( + chat_template_kwargs.pop("tokenize", False) + or isinstance(renderer.tokenizer, MistralTokenizer) + ), **chat_template_kwargs, ) From c27a8b989edd9d3aa7e48c60d02dd58422b00f64 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 19 Jan 2026 09:40:27 +0000 Subject: [PATCH 63/65] Fix Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/chat_completion/serving.py | 4 ++-- vllm/entrypoints/openai/completion/serving.py | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/vllm/entrypoints/openai/chat_completion/serving.py b/vllm/entrypoints/openai/chat_completion/serving.py index a10b53d35a41..a15c99c24c28 100644 --- a/vllm/entrypoints/openai/chat_completion/serving.py +++ b/vllm/entrypoints/openai/chat_completion/serving.py @@ -365,8 +365,6 @@ async def create_chat_completion( ) model_name = self.models.model_name(lora_request) - - tokenizer = await self.engine_client.get_tokenizer() except (ValueError, TypeError, RuntimeError) as e: logger.exception("Error preparing request components") return self.create_error_response(e) @@ -463,6 +461,8 @@ async def create_chat_completion( (result_generator,) = generators # Streaming response + tokenizer = self.renderer.tokenizer + if request.stream: return self.chat_completion_stream_generator( request, diff --git a/vllm/entrypoints/openai/completion/serving.py b/vllm/entrypoints/openai/completion/serving.py index 826047bd8be9..fb14a2307ecf 100644 --- a/vllm/entrypoints/openai/completion/serving.py +++ b/vllm/entrypoints/openai/completion/serving.py @@ -158,11 +158,6 @@ async def create_completion( try: lora_request = self._maybe_get_adapters(request) - - if self.model_config.skip_tokenizer_init: - tokenizer = None - else: - tokenizer = await self.engine_client.get_tokenizer() except (ValueError, TypeError, RuntimeError) as e: logger.exception("Error preparing request components") return self.create_error_response(e) From 8dc62d90235ba7e54423d9b6c7f2efa2d9eb62b4 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 19 Jan 2026 09:52:21 +0000 Subject: [PATCH 64/65] More fixes Signed-off-by: DarkLight1337 --- vllm/entrypoints/openai/engine/serving.py | 2 +- vllm/entrypoints/pooling/__init__.py | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/vllm/entrypoints/openai/engine/serving.py b/vllm/entrypoints/openai/engine/serving.py index 13cfa7b8a300..af4580e7c15a 100644 --- a/vllm/entrypoints/openai/engine/serving.py +++ b/vllm/entrypoints/openai/engine/serving.py @@ -1216,7 +1216,7 @@ async def _preprocess_chat( else: self._validate_input( request=request, - input_ids=engine_prompt["prompt_token_ids"], + input_ids=engine_prompt["prompt_token_ids"], # type: ignore input_text="", ) diff --git a/vllm/entrypoints/pooling/__init__.py b/vllm/entrypoints/pooling/__init__.py index e9b2139b15bb..408542dfa52c 100644 --- a/vllm/entrypoints/pooling/__init__.py +++ b/vllm/entrypoints/pooling/__init__.py @@ -28,21 +28,17 @@ def register_pooling_api_routers(app: FastAPI): async def init_pooling_state( engine_client: "EngineClient", state: "State", args: "Namespace" ): + from vllm.entrypoints.chat_utils import load_chat_template from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.pooling.classify.serving import ServingClassification from vllm.entrypoints.pooling.embed.serving import OpenAIServingEmbedding from vllm.entrypoints.pooling.pooling.serving import OpenAIServingPooling from vllm.entrypoints.pooling.score.serving import ServingScores - from vllm.entrypoints.utils import process_chat_template from vllm.tasks import POOLING_TASKS supported_tasks = await engine_client.get_supported_tasks() - vllm_config = engine_client.vllm_config - - resolved_chat_template = await process_chat_template( - args.chat_template, engine_client, vllm_config.model_config - ) + resolved_chat_template = load_chat_template(args.chat_template) if args.enable_log_requests: request_logger = RequestLogger(max_log_len=args.max_log_len) From 4d6010bcdae22faa3eb4f77b883a99774e64b031 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Mon, 19 Jan 2026 13:34:37 +0000 Subject: [PATCH 65/65] Fix Signed-off-by: DarkLight1337 --- tests/entrypoints/openai/test_serving_chat.py | 90 ++++++++----------- 1 file changed, 39 insertions(+), 51 deletions(-) diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 20a0f16df732..0f8de3435575 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -11,7 +11,7 @@ from openai import OpenAI from vllm._aiter_ops import is_aiter_found_and_supported -from vllm.config.multimodal import MultiModalConfig +from vllm.config import MultiModalConfig from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ChatCompletionResponse, @@ -23,9 +23,12 @@ ) from vllm.entrypoints.openai.models.serving import BaseModelPath, OpenAIServingModels from vllm.entrypoints.openai.parser.harmony_utils import get_encoding +from vllm.inputs import TokensPrompt from vllm.outputs import CompletionOutput, RequestOutput from vllm.renderers.hf import HfRenderer +from vllm.renderers.mistral import MistralRenderer from vllm.tokenizers import get_tokenizer +from vllm.tokenizers.mistral import MistralTokenizer from vllm.tokenizers.registry import tokenizer_args_from_config from vllm.tool_parsers import ToolParserManager from vllm.v1.engine.async_llm import AsyncLLM @@ -105,15 +108,16 @@ def gptoss_server(default_server_args: list[str]): @pytest.fixture(scope="class") def gptoss_speculative_server(default_server_args: list[str]): + attention_backend = ( + "TRITON_ATTN" + if not is_aiter_found_and_supported() + else "ROCM_AITER_UNIFIED_ATTN" + ) server_args = default_server_args + [ "--speculative-config", f'{{"model": "{GPT_OSS_SPECULATOR_NAME}", ' f'"method": "eagle3", "num_speculative_tokens": 3}}', - f"--attention-backend={ - 'TRITON_ATTN' - if not is_aiter_found_and_supported() - else 'ROCM_AITER_UNIFIED_ATTN' - }", + f"--attention-backend={attention_backend}", ] # gpt-oss requires AITER unified attention on ROCm # TODO: Remove after fixing TRITON_ATTN issue on ROCm @@ -522,7 +526,7 @@ class MockModelConfig: encoder_config = None generation_config: str = "auto" media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict) - skip_tokenizer_init = False + skip_tokenizer_init: bool = False def get_diff_sampling_param(self): return self.diff_sampling_param or {} @@ -744,42 +748,32 @@ async def test_serving_chat_should_set_correct_max_tokens(): @pytest.mark.asyncio -async def test_serving_chat_mistral_token_ids_prompt_is_validated(monkeypatch_module): +async def test_serving_chat_mistral_token_ids_prompt_is_validated(): """Regression test: when the Mistral tokenizer path returns token IDs directly, we must still apply input length + max_tokens validation. """ mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False - mock_engine.model_config = MockModelConfig() + mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True) mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - class DummyMistralTokenizer: - def decode(self, token_ids): - # Only used for logging/validation error messages. - return "dummy" - - dummy_tokenizer = DummyMistralTokenizer() - mock_engine.get_tokenizer.return_value = dummy_tokenizer - - # Patch the OpenAI engine serving module to treat our dummy tokenizer - # as a MistralTokenizer. This forces the code path where chat template - # rendering can return a list[int] (token IDs). - import vllm.entrypoints.openai.engine.serving as engine_serving - - monkeypatch_module.setattr( - engine_serving, "MistralTokenizer", DummyMistralTokenizer - ) - - serving_chat = _build_serving_chat(mock_engine) - + mock_tokenizer = MagicMock(spec=MistralTokenizer) + mock_renderer = MistralRenderer(mock_engine.model_config, tokenizer_kwargs={}) + mock_renderer._tokenizer = mock_tokenizer # Force the Mistral chat template renderer to return token IDs. # Choose a prompt length that is < max_model_len, but large enough that # adding max_tokens should exceed the model context window. - serving_chat._apply_mistral_chat_template_async = AsyncMock( - return_value=list(range(95)) + mock_renderer.render_messages_async = AsyncMock( + return_value=( + [], + TokensPrompt(prompt_token_ids=list(range(95))), + ) ) + mock_engine.renderer = mock_renderer + + serving_chat = _build_serving_chat(mock_engine) req = ChatCompletionRequest( model=MODEL_NAME, @@ -793,39 +787,33 @@ def decode(self, token_ids): @pytest.mark.asyncio -async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected( - monkeypatch_module, -): +async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected(): """Regression test: MistralTokenizer token-id prompts must still enforce the max context length for the input itself (token_num >= max_model_len). """ mock_engine = MagicMock(spec=AsyncLLM) mock_engine.errored = False - mock_engine.model_config = MockModelConfig() + mock_engine.model_config = MockModelConfig(skip_tokenizer_init=True) mock_engine.input_processor = MagicMock() mock_engine.io_processor = MagicMock() - class DummyMistralTokenizer: - def decode(self, token_ids): - return "dummy" - - dummy_tokenizer = DummyMistralTokenizer() - mock_engine.get_tokenizer.return_value = dummy_tokenizer - - import vllm.entrypoints.openai.engine.serving as engine_serving - - monkeypatch_module.setattr( - engine_serving, "MistralTokenizer", DummyMistralTokenizer - ) - - serving_chat = _build_serving_chat(mock_engine) - + mock_tokenizer = MagicMock(spec=MistralTokenizer) + mock_renderer = MistralRenderer(mock_engine.model_config, tokenizer_kwargs={}) + mock_renderer._tokenizer = mock_tokenizer # prompt_token_ids length == max_model_len should be rejected for # completion-like requests (ChatCompletionRequest). - serving_chat._apply_mistral_chat_template_async = AsyncMock( - return_value=list(range(mock_engine.model_config.max_model_len)) + mock_renderer.render_messages_async = AsyncMock( + return_value=( + [], + TokensPrompt( + prompt_token_ids=list(range(mock_engine.model_config.max_model_len)) + ), + ) ) + mock_engine.renderer = mock_renderer + + serving_chat = _build_serving_chat(mock_engine) req = ChatCompletionRequest( model=MODEL_NAME,