From a019fe63f303695f8a288763140e076939157db4 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 09:26:49 +0000 Subject: [PATCH 01/23] [Renderer] Move InputPreprocessor into Renderer (1/2) Signed-off-by: DarkLight1337 --- tests/v1/e2e/test_streaming_input.py | 2 +- .../test_async_llm_streaming.py | 2 +- vllm/benchmarks/mm_processor.py | 90 ++++++++++++- vllm/engine/protocol.py | 15 ++- vllm/entrypoints/llm.py | 43 +++--- vllm/entrypoints/openai/realtime/serving.py | 4 +- vllm/inputs/__init__.py | 2 - vllm/inputs/data.py | 15 --- vllm/inputs/preprocess.py | 114 ++++------------ vllm/model_executor/models/clip.py | 10 ++ vllm/model_executor/models/lfm2_vl.py | 10 ++ vllm/model_executor/models/mllama4.py | 10 ++ .../model_executor/models/nano_nemotron_vl.py | 10 ++ vllm/model_executor/models/nemotron_parse.py | 10 ++ vllm/model_executor/models/ovis.py | 10 ++ vllm/model_executor/models/ovis2_5.py | 10 ++ vllm/model_executor/models/paligemma.py | 10 ++ vllm/model_executor/models/siglip.py | 10 ++ vllm/model_executor/models/ultravox.py | 10 ++ .../model_executor/models/voxtral_realtime.py | 3 +- vllm/model_executor/models/whisper.py | 13 ++ vllm/multimodal/processing/context.py | 120 +++-------------- vllm/renderers/base.py | 126 +++++++++++++++++- vllm/renderers/params.py | 12 +- vllm/v1/engine/async_llm.py | 6 +- vllm/v1/engine/input_processor.py | 24 ++-- vllm/v1/metrics/stats.py | 6 + 27 files changed, 421 insertions(+), 276 deletions(-) diff --git a/tests/v1/e2e/test_streaming_input.py b/tests/v1/e2e/test_streaming_input.py index 4c9b43099e4a..01c5fe6f8eb0 100644 --- a/tests/v1/e2e/test_streaming_input.py +++ b/tests/v1/e2e/test_streaming_input.py @@ -19,7 +19,7 @@ import pytest_asyncio from vllm import SamplingParams -from vllm.inputs import StreamingInput +from vllm.engine.protocol import StreamingInput from vllm.outputs import RequestOutput from vllm.platforms import current_platform from vllm.sampling_params import RequestOutputKind diff --git a/tests/v1/streaming_input/test_async_llm_streaming.py b/tests/v1/streaming_input/test_async_llm_streaming.py index b5ba757d0a91..b532eed15f38 100644 --- a/tests/v1/streaming_input/test_async_llm_streaming.py +++ b/tests/v1/streaming_input/test_async_llm_streaming.py @@ -7,7 +7,7 @@ import pytest -from vllm.inputs import StreamingInput +from vllm.engine.protocol import StreamingInput from vllm.outputs import RequestOutput from vllm.sampling_params import RequestOutputKind, SamplingParams from vllm.v1.engine.async_llm import AsyncLLM diff --git a/vllm/benchmarks/mm_processor.py b/vllm/benchmarks/mm_processor.py index b7dc3bebc6d8..4d372da89e74 100644 --- a/vllm/benchmarks/mm_processor.py +++ b/vllm/benchmarks/mm_processor.py @@ -28,11 +28,9 @@ ) from vllm.benchmarks.throughput import get_requests from vllm.engine.arg_utils import EngineArgs -from vllm.multimodal.processing.context import ( - get_timing_stats_from_engine_client, -) from vllm.utils.gc_utils import freeze_gc_heap from vllm.utils.import_utils import PlaceholderModule +from vllm.v1.engine.llm_engine import LLMEngine try: import pandas as pd @@ -40,15 +38,97 @@ pd = PlaceholderModule("pandas") +def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]: + """ + Get all multimodal timing stats from the LLM engine. + + Collects both preprocessing stats (HF processor, hashing, cache lookup, + prompt update) and encoder forward pass timing, merged by request_id. + + Args: + engine: The LLM engine (has input_processor and workers). + + Returns: + Dictionary mapping request_id to merged stats dict containing + both preprocessing and encoder timing metrics. + + Example: + { + 'request-123': { + 'hf_processor_time': 0.45, + 'hashing_time': 0.02, + 'cache_lookup_time': 0.01, + 'prompt_update_time': 0.03, + 'preprocessor_total_time': 0.51, + 'encoder_forward_time': 0.23, + 'num_encoder_calls': 1 + } + } + """ + observability_config = llm_engine.vllm_config.observability_config + if not observability_config or not observability_config.enable_mm_processor_stats: + return {} + + renderer = llm_engine.renderer + mm_processor = renderer.get_mm_processor() + preprocessing_stats = mm_processor.info.ctx.get_all_timing_stats() + + encoder_stats = dict[str, dict[str, float]]() + for worker_stats in llm_engine.collective_rpc("get_encoder_timing_stats"): + if not worker_stats: + continue + + for request_id, stats_dict in worker_stats.items(): + if request_id not in encoder_stats: + encoder_stats[request_id] = dict(stats_dict) + else: + # Aggregate timing metrics across workers + current_time = encoder_stats[request_id].get( + "encoder_forward_time", 0.0 + ) + new_time = stats_dict.get("encoder_forward_time", 0.0) + encoder_stats[request_id]["encoder_forward_time"] = max( + current_time, new_time + ) + + current_calls = encoder_stats[request_id].get("num_encoder_calls", 0) + new_calls = stats_dict.get("num_encoder_calls", 0) + encoder_stats[request_id]["num_encoder_calls"] = max( + current_calls, new_calls + ) + + merged_stats = dict[str, dict[str, float]]() + + for request_id, prep_dict in preprocessing_stats.items(): + merged_stats[request_id] = dict(prep_dict) + + for request_id, enc_dict in encoder_stats.items(): + if request_id in merged_stats: + merged_stats[request_id].update(enc_dict) + continue + + # In V1 engine, the request_id in encoder_stats has a suffix + # appended to the original request_id (which is used in + # preprocessing_stats). + # We try to strip the suffix to find the matching request. + possible_original_id = request_id.rpartition("-")[0] + if possible_original_id and possible_original_id in merged_stats: + merged_stats[possible_original_id].update(enc_dict) + else: + merged_stats[request_id] = dict(enc_dict) + + return merged_stats + + def collect_mm_processor_stats( - llm_engine: Any, + llm_engine: LLMEngine, num_warmup_reqs: int = 0, ) -> dict[str, list[float]]: """ Collect multimodal processor timing stats. Returns a dictionary mapping stage names to lists of timing values (in seconds). """ - all_stats = get_timing_stats_from_engine_client(llm_engine) + all_stats = get_timing_stats_from_engine(llm_engine) stat_keys = [ "hf_processor_time", diff --git a/vllm/engine/protocol.py b/vllm/engine/protocol.py index 0f2e62c59e89..365cfb50beda 100644 --- a/vllm/engine/protocol.py +++ b/vllm/engine/protocol.py @@ -3,6 +3,7 @@ from abc import ABC, abstractmethod from collections.abc import AsyncGenerator, Iterable, Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Any from vllm.config import ModelConfig, VllmConfig @@ -10,7 +11,7 @@ WeightTransferInitRequest, WeightTransferUpdateRequest, ) -from vllm.inputs.data import PromptType, StreamingInput +from vllm.inputs.data import PromptType from vllm.lora.request import LoRARequest from vllm.outputs import PoolingRequestOutput, RequestOutput from vllm.plugins.io_processors import IOProcessor @@ -26,6 +27,18 @@ from vllm.v1.engine import PauseMode +@dataclass +class StreamingInput: + """Input data for a streaming generation request. + + This is used with generate() to support multi-turn streaming sessions + where inputs are provided via an async generator. + """ + + prompt: PromptType + sampling_params: SamplingParams | None = None + + class EngineClient(ABC): """Protocol class for Clients to Engine""" diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index 9474c543e6f0..f3c4d7540c5d 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -72,7 +72,7 @@ ) from vllm.platforms import current_platform from vllm.pooling_params import PoolingParams -from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs +from vllm.renderers import ChatParams, merge_kwargs from vllm.renderers.inputs import DictPrompt, TokPrompt from vllm.renderers.inputs.preprocess import ( conversation_to_seq, @@ -876,19 +876,6 @@ def create_tokens_prompt_from_beam(beam: BeamSearchSequence) -> TokensPrompt: return outputs - def _get_cmpl_tok_params(self, tokenization_kwargs: dict[str, Any] | None): - model_config = self.model_config - encoder_config = model_config.encoder_config or {} - - return TokenizeParams( - max_total_tokens=model_config.max_model_len, - do_lower_case=encoder_config.get("do_lower_case", False), - # For Whisper, special tokens should be provided by the user based - # on the task and language of their request. Also needed to avoid - # appending an EOS token to the prompt which disrupts generation. - add_special_tokens=not model_config.is_encoder_decoder, - ).with_kwargs(tokenization_kwargs) - def _preprocess_cmpl( self, prompts: Sequence[PromptType], @@ -910,20 +897,12 @@ def _preprocess_cmpl( parsed_prompts = [ parse_model_prompt(model_config, prompt) for prompt in prompts ] - tok_params = self._get_cmpl_tok_params(tokenization_kwargs) + tok_params = renderer.default_cmpl_tok_params.with_kwargs( + **(tokenization_kwargs or {}) + ) return renderer.render_cmpl(parsed_prompts, tok_params) - def _get_chat_tok_params(self, tokenization_kwargs: dict[str, Any] | None): - model_config = self.model_config - encoder_config = model_config.encoder_config or {} - - return TokenizeParams( - max_total_tokens=model_config.max_model_len, - do_lower_case=encoder_config.get("do_lower_case", False), - add_special_tokens=False, - ).with_kwargs(tokenization_kwargs) - def _preprocess_chat( self, conversations: Sequence[list[ChatCompletionMessageParam]], @@ -961,7 +940,9 @@ def _preprocess_chat( ), ), ) - tok_params = self._get_chat_tok_params(tokenization_kwargs) + tok_params = renderer.default_chat_tok_params.with_kwargs( + **(tokenization_kwargs or {}) + ) _, engine_prompts = renderer.render_chat( conversations, @@ -1653,7 +1634,10 @@ def score( architecture=architecture, ) - tok_params = self._get_cmpl_tok_params(tokenization_kwargs) + renderer = self.renderer + tok_params = renderer.default_cmpl_tok_params.with_kwargs( + **(tokenization_kwargs or {}) + ) encode_kwargs = tok_params.get_encode_kwargs() if model_config.is_cross_encoder: @@ -1970,7 +1954,10 @@ def _add_request( dict(truncate_prompt_tokens=params.truncate_prompt_tokens), ) - tok_params = self._get_cmpl_tok_params(tokenization_kwargs) + renderer = self.renderer + tok_params = renderer.default_cmpl_tok_params.with_kwargs( + **(tokenization_kwargs or {}) + ) tokenization_kwargs = tok_params.get_encode_kwargs() engine_request = self.input_processor.process_inputs( diff --git a/vllm/entrypoints/openai/realtime/serving.py b/vllm/entrypoints/openai/realtime/serving.py index 8a2d62a37241..f83ab9e6c006 100644 --- a/vllm/entrypoints/openai/realtime/serving.py +++ b/vllm/entrypoints/openai/realtime/serving.py @@ -8,11 +8,11 @@ import numpy as np -from vllm.engine.protocol import EngineClient +from vllm.engine.protocol import EngineClient, StreamingInput from vllm.entrypoints.logger import RequestLogger from vllm.entrypoints.openai.engine.serving import OpenAIServing from vllm.entrypoints.openai.models.serving import OpenAIServingModels -from vllm.inputs.data import PromptType, StreamingInput +from vllm.inputs.data import PromptType from vllm.logger import init_logger from vllm.model_executor.models.interfaces import SupportsRealtime diff --git a/vllm/inputs/__init__.py b/vllm/inputs/__init__.py index de8ddc615f8f..2f9db8bdd9ca 100644 --- a/vllm/inputs/__init__.py +++ b/vllm/inputs/__init__.py @@ -12,7 +12,6 @@ PromptType, SingletonInputs, SingletonPrompt, - StreamingInput, TextPrompt, TokenInputs, TokensPrompt, @@ -36,5 +35,4 @@ "EncoderDecoderInputs", "ProcessorInputs", "SingletonInputs", - "StreamingInput", ] diff --git a/vllm/inputs/data.py b/vllm/inputs/data.py index 157ab337e1e0..4f1b3b9ca64b 100644 --- a/vllm/inputs/data.py +++ b/vllm/inputs/data.py @@ -1,13 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal, TypeAlias import torch from typing_extensions import NotRequired, TypedDict -from vllm.sampling_params import SamplingParams - if TYPE_CHECKING: from vllm.multimodal.inputs import ( MultiModalDataDict, @@ -299,15 +296,3 @@ class EncoderDecoderInputs(TypedDict): SingletonInputs: TypeAlias = DecoderOnlyInputs | MultiModalEncDecInputs """The inputs for a single encoder/decoder prompt.""" - - -@dataclass -class StreamingInput: - """Input data for a streaming generation request. - - This is used with generate() to support multi-turn streaming sessions - where inputs are provided via an async generator. - """ - - prompt: PromptType - sampling_params: SamplingParams | None = None diff --git a/vllm/inputs/preprocess.py b/vllm/inputs/preprocess.py index ef1f2e0bf6f0..08a37b6da1e6 100644 --- a/vllm/inputs/preprocess.py +++ b/vllm/inputs/preprocess.py @@ -9,13 +9,11 @@ from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry -from vllm.multimodal.cache import BaseMultiModalProcessorCache from vllm.multimodal.inputs import ( MultiModalDataDict, MultiModalInputs, MultiModalUUIDDict, ) -from vllm.multimodal.processing import BaseMultiModalProcessor from vllm.renderers import BaseRenderer, renderer_from_config from vllm.renderers.inputs import ( DecoderDictPrompt, @@ -28,8 +26,6 @@ ) from vllm.renderers.inputs.preprocess import parse_dec_only_prompt, parse_enc_dec_prompt from vllm.tokenizers import TokenizerLike -from vllm.utils.jsontree import json_iter_leaves -from vllm.v1.metrics.stats import MultiModalCacheStats from .data import ( DecoderInputs, @@ -57,17 +53,12 @@ def __init__( vllm_config: VllmConfig, renderer: BaseRenderer | None = None, mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY, - mm_processor_cache: BaseMultiModalProcessorCache | None = None, ) -> None: super().__init__() self.model_config = vllm_config.model_config - self.observability_config = vllm_config.observability_config self.renderer = renderer or renderer_from_config(vllm_config) self.mm_registry = mm_registry - self.mm_processor_cache = mm_processor_cache - - self.mm_cache_stats = MultiModalCacheStats() if mm_processor_cache else None @property def tokenizer(self) -> TokenizerLike | None: @@ -124,23 +115,6 @@ def _prepare_decoder_input_ids(self, decoder_input_ids: list[int]) -> list[int]: return decoder_input_ids - def _get_tokenization_kw( - self, - overrides: dict[str, Any] | None = None, - ) -> dict[str, Any]: - kwargs = dict[str, Any]() - - if self.model_config.is_encoder_decoder: - # For Whisper, special tokens should be provided by the user based - # on the task and language of their request. Also needed to avoid - # appending an EOS token to the prompt which disrupts generation. - kwargs["add_special_tokens"] = False - - if overrides: - kwargs.update(overrides) - - return kwargs - def _tokenize_prompt( self, prompt: str, @@ -150,26 +124,18 @@ def _tokenize_prompt( Apply the model's tokenizer to a text prompt, returning the corresponding token IDs. """ - tokenizer = self.get_tokenizer() - tokenization_kwargs = self._get_tokenization_kw(tokenization_kwargs) - - encoder_config = self.model_config.encoder_config + renderer = self.renderer - if encoder_config and encoder_config.get("do_lower_case", False): - prompt = prompt.lower() - - return tokenizer.encode(prompt, **tokenization_kwargs) + tok_params = renderer.default_cmpl_tok_params.with_kwargs( + **(tokenization_kwargs or {}) + ) - def _get_mm_processor(self) -> BaseMultiModalProcessor: - if not hasattr(self, "_mm_processor"): - self._mm_processor = self.mm_registry.create_processor( - self.model_config, - self.observability_config, - tokenizer=self.tokenizer, - cache=self.mm_processor_cache, - ) + tok_prompt = renderer.tokenize_prompt( + TextPrompt(prompt=prompt), + tok_params, + ) - return self._mm_processor + return tok_prompt["prompt_token_ids"] def _process_multimodal( self, @@ -184,33 +150,20 @@ def _process_multimodal( Apply the model's multi-modal processor to a multi-modal prompt, returning the corresponding token IDs and metadata. """ - mm_processor = self._get_mm_processor() + mm_processor = self.renderer.get_mm_processor() if mm_processor_kwargs is None: mm_processor_kwargs = {} mm_items = mm_processor.info.parse_mm_data(mm_data) - mm_input = mm_processor.apply( + + return mm_processor.apply( prompt, mm_items, hf_processor_mm_kwargs=mm_processor_kwargs, tokenization_kwargs=tokenization_kwargs, mm_uuids=mm_uuids, ) - mm_hashes = mm_input["mm_hashes"] - - # Validate that all mm items have a string as their hash - contains_only_strings = all( - isinstance(leaf, str) for leaf in json_iter_leaves(mm_hashes) - ) - if not contains_only_strings: - raise ValueError( - f"mm_hashes must contain only strings, got: {mm_hashes}. " - "This is likely due to an incorrect custom implementation of " - "MultiModalProcessor.apply method." - ) - - return mm_input def _process_embeds( self, @@ -245,19 +198,18 @@ def _process_embeds( def _truncate_inputs( self, inputs: list[int], tokenization_kwargs: dict[str, Any] | None = None ) -> list[int]: - if ( - not tokenization_kwargs - or "truncation" not in tokenization_kwargs - or self.tokenizer is None - ): - return inputs + renderer = self.renderer - max_length = tokenization_kwargs["max_length"] + tok_params = renderer.default_cmpl_tok_params.with_kwargs( + **(tokenization_kwargs or {}) + ) - if self.tokenizer.truncation_side == "left": - return inputs[-max_length:] - else: - return inputs[:max_length] + tok_prompt = renderer.tokenize_prompt( + TokensPrompt(prompt_token_ids=inputs), + tok_params, + ) + + return tok_prompt["prompt_token_ids"] def _process_tokens( self, @@ -539,26 +491,6 @@ def preprocess( """Preprocess the input prompt.""" res = self._preprocess(prompt, tokenization_kwargs, mm_uuids=mm_uuids) - if self.mm_processor_cache and self.mm_cache_stats is not None: - delta = self.mm_processor_cache.make_stats(delta=True) - self.mm_cache_stats.requests += 1 - self.mm_cache_stats.queries += delta.total - self.mm_cache_stats.hits += delta.hits + self.renderer.update_mm_cache_stats() return res - - def stat_mm_cache(self) -> MultiModalCacheStats | None: - mm_cache_stats = self.mm_cache_stats - if mm_cache_stats is None: - return None - - self.mm_cache_stats = MultiModalCacheStats() - - return mm_cache_stats - - def clear_mm_cache(self) -> None: - if self.mm_processor_cache is not None: - self.mm_processor_cache.clear_cache() - - if self.mm_cache_stats is not None: - self.mm_cache_stats.reset = True diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 3f189eacce7a..f27a083c224b 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -47,6 +47,7 @@ PromptReplacement, PromptUpdate, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -126,6 +127,15 @@ def get_vision_encoder_info(self): def get_hf_processor(self, **kwargs: object): return self.ctx.get_hf_processor(CLIPProcessor, **kwargs) + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": 1} diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index 98fd0b1b0c5b..1b282fc4dd6e 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -42,6 +42,7 @@ PromptReplacement, PromptUpdateDetails, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -90,6 +91,15 @@ def get_hf_processor(self, **kwargs): def get_image_processor(self, **kwargs: object) -> Lfm2VlImageProcessorFast: return self.get_hf_processor(**kwargs).image_processor + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": None} diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 3752a77041d9..6885f02774a0 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -66,6 +66,7 @@ PromptUpdate, PromptUpdateDetails, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -554,6 +555,15 @@ def get_hf_processor(self, **kwargs: object) -> Llama4Processor: Llama4Processor, use_fast=kwargs.pop("use_fast", True), **kwargs ) + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_supported_mm_limits(self) -> Mapping[str, int | None]: # Although vLLM can support more images from an infra capability # perspective, we do not recommend using >10 images in practice. diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index fb683487f44d..93d896feb04d 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -76,6 +76,7 @@ PromptUpdateDetails, _seq2tokens, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.tokenizers import TokenizerLike, cached_tokenizer_from_config from vllm.transformers_utils.configs.radio import RadioConfig @@ -1093,6 +1094,15 @@ def get_hf_processor( ) -> BaseNanoNemotronVLProcessor: raise NotImplementedError + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": None} diff --git a/vllm/model_executor/models/nemotron_parse.py b/vllm/model_executor/models/nemotron_parse.py index b94b606a109c..5be4db56e851 100644 --- a/vllm/model_executor/models/nemotron_parse.py +++ b/vllm/model_executor/models/nemotron_parse.py @@ -58,6 +58,7 @@ PromptReplacement, PromptUpdate, ) +from vllm.renderers import TokenizeParams from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.configs.radio import RadioConfig from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -608,6 +609,15 @@ def get_hf_processor(self, **kwargs) -> NemotronParseProcessor: **kwargs, ) + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + @property def skip_prompt_length_check(self) -> bool: return True # Because the encoder prompt is padded diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index 7e02d87ec5b2..d7f419cdcaec 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -53,6 +53,7 @@ BaseProcessingInfo, PromptReplacement, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.ovis import OvisProcessor from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -264,6 +265,15 @@ def get_hf_processor(self, **kwargs: object): **kwargs, ) + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_image_segment_len(self) -> int: visual_tokenizer_config = self.get_hf_config().visual_tokenizer_config image_size = visual_tokenizer_config.backbone_config.image_size diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index 00418d707119..abee3a6bb821 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -35,6 +35,7 @@ BaseProcessingInfo, PromptReplacement, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processors.ovis2_5 import Ovis2_5Processor from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -183,6 +184,15 @@ def get_hf_processor(self, **kwargs): temporal_patch_size=vit_config.temporal_patch_size, ) + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_image_processor(self) -> BaseImageProcessor: return self.get_hf_processor().image_processor # type: ignore diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py index e551f9fc99fe..d68d4db702cf 100644 --- a/vllm/model_executor/models/paligemma.py +++ b/vllm/model_executor/models/paligemma.py @@ -32,6 +32,7 @@ PromptUpdate, PromptUpdateDetails, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -102,6 +103,15 @@ def get_hf_config(self): def get_vision_encoder_info(self): return get_vision_encoder_info(self.get_hf_config()) + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": 1} diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 92ecc75793ce..5060492769d2 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -53,6 +53,7 @@ PromptReplacement, PromptUpdate, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -111,6 +112,15 @@ def get_vision_encoder_info(self): def get_hf_processor(self, **kwargs: object): return self.ctx.get_hf_processor(SiglipProcessor, **kwargs) + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": 1} diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index d7a9bd4fd903..156c985f5535 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -42,6 +42,7 @@ PromptReplacement, PromptUpdate, ) +from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.ultravox import UltravoxConfig from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -133,6 +134,15 @@ def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor: assert isinstance(feature_extractor, WhisperFeatureExtractor) return feature_extractor + def get_default_tok_params(self) -> TokenizeParams: + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_data_parser(self): feature_extractor = self.get_feature_extractor() diff --git a/vllm/model_executor/models/voxtral_realtime.py b/vllm/model_executor/models/voxtral_realtime.py index 81406c66b084..726f67096d1e 100644 --- a/vllm/model_executor/models/voxtral_realtime.py +++ b/vllm/model_executor/models/voxtral_realtime.py @@ -17,8 +17,9 @@ from vllm.compilation.decorators import support_torch_compile from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig +from vllm.engine.protocol import StreamingInput from vllm.envs import VLLM_ENGINE_ITERATION_TIMEOUT_S -from vllm.inputs.data import PromptType, StreamingInput, TokensPrompt +from vllm.inputs.data import PromptType, TokensPrompt from vllm.logger import init_logger from vllm.model_executor.models.interfaces import MultiModalEmbeddings, SupportsRealtime from vllm.model_executor.models.voxtral import ( diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index 26c7b62e8a6a..879f332a924c 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -55,6 +55,7 @@ PromptReplacement, PromptUpdate, ) +from vllm.renderers import TokenizeParams from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.jsontree import json_map_leaves from vllm.utils.tensor_schema import TensorSchema, TensorShape @@ -644,6 +645,18 @@ class WhisperProcessingInfo(BaseProcessingInfo): def get_hf_config(self) -> WhisperConfig: return self.ctx.get_hf_config(WhisperConfig) + def get_default_tok_params(self) -> TokenizeParams: + # Special tokens should be provided by the user based on the + # task and language of their request. Also needed to avoid + # appending an EOS token to the prompt which disrupts generation. + return ( + super() + .get_default_tok_params() + .with_kwargs( + add_special_tokens=False, + ) + ) + def get_data_parser(self): feature_extractor = self.get_feature_extractor() diff --git a/vllm/multimodal/processing/context.py b/vllm/multimodal/processing/context.py index 34a1867109c0..b131ee3c49a8 100644 --- a/vllm/multimodal/processing/context.py +++ b/vllm/multimodal/processing/context.py @@ -21,6 +21,7 @@ MultiModalDataItems, MultiModalDataParser, ) +from vllm.renderers import TokenizeParams from vllm.tokenizers import TokenizerLike from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.func_utils import get_allowed_kwarg_only_overrides @@ -93,110 +94,6 @@ def to_dict(self) -> dict[str, float]: } -def get_timing_stats_from_engine_client( - engine_client: Any, -) -> dict[str, dict[str, float]]: - """ - Get all multimodal timing stats from the engine client. - - Collects both preprocessing stats (HF processor, hashing, cache lookup, - prompt update) and encoder forward pass timing, merged by request_id. - - Args: - engine_client: The engine client (has input_processor and workers). - - Returns: - Dictionary mapping request_id to merged stats dict containing - both preprocessing and encoder timing metrics. - - Example: - { - 'request-123': { - 'hf_processor_time': 0.45, - 'hashing_time': 0.02, - 'cache_lookup_time': 0.01, - 'prompt_update_time': 0.03, - 'preprocessor_total_time': 0.51, - 'encoder_forward_time': 0.23, - 'num_encoder_calls': 1 - } - } - """ - try: - if not engine_client.vllm_config.observability_config.enable_mm_processor_stats: - return {} - except (AttributeError, RuntimeError): - return {} - - preprocessing_stats = {} - try: - input_processor = engine_client.input_processor - input_preprocessor = input_processor.input_preprocessor - - if hasattr(input_preprocessor, "_get_mm_processor"): - mm_processor = input_preprocessor._get_mm_processor() - if mm_processor is not None and hasattr(mm_processor, "info"): - ctx = mm_processor.info.ctx - preprocessing_stats = ctx.get_all_timing_stats() - except (AttributeError, RuntimeError): - pass - - encoder_stats = {} - try: - if hasattr(engine_client, "collective_rpc"): - encoder_stats_results = engine_client.collective_rpc( - "get_encoder_timing_stats" - ) - if encoder_stats_results and len(encoder_stats_results) > 0: - for worker_stats in encoder_stats_results: - if not worker_stats: - continue - for request_id, stats_dict in worker_stats.items(): - if request_id not in encoder_stats: - encoder_stats[request_id] = dict(stats_dict) - else: - # Aggregate timing metrics across workers - current_time = encoder_stats[request_id].get( - "encoder_forward_time", 0.0 - ) - new_time = stats_dict.get("encoder_forward_time", 0.0) - encoder_stats[request_id]["encoder_forward_time"] = max( - current_time, new_time - ) - - current_calls = encoder_stats[request_id].get( - "num_encoder_calls", 0 - ) - new_calls = stats_dict.get("num_encoder_calls", 0) - encoder_stats[request_id]["num_encoder_calls"] = max( - current_calls, new_calls - ) - except (AttributeError, RuntimeError): - pass - - merged_stats = {} - - for request_id, prep_dict in preprocessing_stats.items(): - merged_stats[request_id] = dict(prep_dict) - - for request_id, enc_dict in encoder_stats.items(): - if request_id in merged_stats: - merged_stats[request_id].update(enc_dict) - continue - - # In V1 engine, the request_id in encoder_stats has a suffix - # appended to the original request_id (which is used in - # preprocessing_stats). - # We try to strip the suffix to find the matching request. - possible_original_id = request_id.rpartition("-")[0] - if possible_original_id and possible_original_id in merged_stats: - merged_stats[possible_original_id].update(enc_dict) - else: - merged_stats[request_id] = dict(enc_dict) - - return merged_stats - - @contextmanager def timed_preprocessor_operation(ctx: "InputProcessingContext", stage_name: str): """ @@ -576,6 +473,21 @@ def get_hf_processor(self, **kwargs: object) -> ProcessorMixin: """ return self.ctx.get_hf_processor(**kwargs) + def get_default_tok_params(self) -> TokenizeParams: + """Construct the default parameters for tokenization.""" + model_config = self.ctx.model_config + encoder_config = model_config.encoder_config or {} + + return TokenizeParams( + max_total_tokens=model_config.max_model_len, + do_lower_case=encoder_config.get("do_lower_case", False), + add_special_tokens=True, + ) + + @cached_property + def default_tok_params(self) -> TokenizeParams: + return self.get_default_tok_params() + def _get_expected_hidden_size(self) -> int | None: """ Get expected hidden size for embedding validation if `mm_embeds` are enabled. diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 05058c5491b8..95a08fa27f0f 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -3,12 +3,14 @@ import asyncio from abc import ABC, abstractmethod from collections.abc import Sequence +from functools import cached_property from typing import TYPE_CHECKING, Any, overload from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import AsyncMicrobatchTokenizer +from vllm.v1.metrics.stats import MultiModalCacheStats from .embed_utils import safe_load_prompt_embeds from .inputs import ( @@ -26,6 +28,8 @@ ChatCompletionMessageParam, ConversationMessage, ) + from vllm.multimodal.cache import BaseMultiModalProcessorCache + from vllm.multimodal.processing import BaseMultiModalProcessor logger = init_logger(__name__) @@ -43,11 +47,16 @@ def from_config( def __init__(self, config: "VllmConfig") -> None: super().__init__() + self.config = config self.model_config = config.model_config # Lazy initialization since offline LLM doesn't use async self._async_tokenizer: AsyncMicrobatchTokenizer | None = None + # Lazy initialization since it's only for MM models + self._mm_processor: BaseMultiModalProcessor | None = None + self._mm_cache_stats: MultiModalCacheStats | None = None + @property @abstractmethod def tokenizer(self) -> TokenizerLike | None: @@ -66,6 +75,69 @@ def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer: return self._async_tokenizer + @cached_property + def mm_processor(self) -> "BaseMultiModalProcessor | None": + vllm_config = self.config + if not vllm_config.model_config.is_multimodal_model: + return None + + return self.get_mm_processor() + + def get_mm_processor(self) -> "BaseMultiModalProcessor": + if self._mm_processor is None: + from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry + + vllm_config = self.config + mm_processor_cache = mm_registry.processor_cache_from_config(vllm_config) + self._mm_processor = mm_registry.create_processor( + vllm_config.model_config, + vllm_config.observability_config, + tokenizer=self.tokenizer, + cache=mm_processor_cache, + ) + self._mm_cache_stats = ( + MultiModalCacheStats() if mm_processor_cache else None + ) + + return self._mm_processor + + def get_mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None": + mm_processor = self.mm_processor + if mm_processor is None: + return None + + return mm_processor.cache + + def stat_mm_cache(self) -> MultiModalCacheStats | None: + mm_cache_stats = self._mm_cache_stats + if mm_cache_stats is None: + return None + + self._mm_cache_stats = MultiModalCacheStats() + + return mm_cache_stats + + def update_mm_cache_stats(self) -> None: + mm_processor_cache = self.get_mm_processor_cache() + mm_cache_stats = self._mm_cache_stats + + if mm_processor_cache and mm_cache_stats: + delta = mm_processor_cache.make_stats(delta=True) + mm_cache_stats.record(delta.total, delta.hits) + + def clear_mm_cache(self) -> None: + mm_processor_cache = self.get_mm_processor_cache() + if mm_processor_cache is not None: + mm_processor_cache.clear_cache() + + if self._mm_cache_stats is not None: + self._mm_cache_stats.reset = True + + def shutdown(self) -> None: + mm_processor_cache = self.get_mm_processor_cache() + if mm_processor_cache is not None: + mm_processor_cache.close() + def get_bos_token_id(self) -> int | None: if self.tokenizer is None: logger.warning_once( @@ -84,6 +156,36 @@ def get_eos_token_id(self) -> int | None: return self.tokenizer.eos_token_id + @cached_property + def default_cmpl_tok_params(self) -> TokenizeParams: + mm_processor = self.get_mm_processor() + if mm_processor is not None: + return mm_processor.info.default_tok_params + + model_config = self.model_config + encoder_config = model_config.encoder_config or {} + + return TokenizeParams( + max_total_tokens=model_config.max_model_len, + do_lower_case=encoder_config.get("do_lower_case", False), + add_special_tokens=True, + ) + + @cached_property + def default_chat_tok_params(self) -> TokenizeParams: + mm_processor = self.get_mm_processor() + if mm_processor is not None: + return mm_processor.info.default_tok_params + + model_config = self.model_config + encoder_config = model_config.encoder_config or {} + + return TokenizeParams( + max_total_tokens=model_config.max_model_len, + do_lower_case=encoder_config.get("do_lower_case", False), + add_special_tokens=False, + ) + # Step 1: Convert raw inputs to prompts def render_prompt( self, @@ -317,10 +419,13 @@ def _apply_prompt_extras( def render_cmpl( self, prompts: Sequence[DictPrompt | bytes], - tok_params: TokenizeParams, + tok_params: TokenizeParams | None = None, *, prompt_extras: dict[str, Any] | None = None, ): + if tok_params is None: + tok_params = self.default_cmpl_tok_params + dict_prompts = self.render_prompts(prompts) # NOTE: Some MM models have non-default `add_special_tokens` @@ -329,6 +434,7 @@ def render_cmpl( self._apply_prompt_extras(dict_prompts, prompt_extras) return dict_prompts + dict_prompts = self.render_prompts(prompts) tok_prompts = self.tokenize_prompts(dict_prompts, tok_params) self._apply_prompt_extras(tok_prompts, prompt_extras) @@ -339,14 +445,14 @@ def render_cmpl( async def render_cmpl_async( self, prompts: Sequence[DictPrompt | bytes], - tok_params: TokenizeParams, + tok_params: TokenizeParams | None = None, *, prompt_extras: dict[str, Any] | None = None, ): - dict_prompts = await self.render_prompts_async(prompts) + if tok_params is None: + tok_params = self.default_cmpl_tok_params - # NOTE: MM data cannot be passed to online Completions API - # so we don't have the special case that is in the offline version + dict_prompts = await self.render_prompts_async(prompts) tok_prompts = await self.tokenize_prompts_async(dict_prompts, tok_params) self._apply_prompt_extras(tok_prompts, prompt_extras) @@ -358,10 +464,13 @@ def render_chat( self, conversations: Sequence[list["ChatCompletionMessageParam"]], chat_params: ChatParams, - tok_params: TokenizeParams, + tok_params: TokenizeParams | None = None, *, prompt_extras: dict[str, Any] | None = None, ): + if tok_params is None: + tok_params = self.default_chat_tok_params + rendered = [ self.render_messages(conversation, chat_params) for conversation in conversations @@ -384,10 +493,13 @@ async def render_chat_async( self, conversations: Sequence[list["ChatCompletionMessageParam"]], chat_params: ChatParams, - tok_params: TokenizeParams, + tok_params: TokenizeParams | None = None, *, prompt_extras: dict[str, Any] | None = None, ): + if tok_params is None: + tok_params = self.default_chat_tok_params + rendered = [ self.render_messages_async(conversation, chat_params) for conversation in conversations diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index a860fcd951f1..52a7b96755db 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -3,7 +3,6 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, TypeVar -from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption from vllm.exceptions import VLLMValidationError from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt from vllm.logger import init_logger @@ -12,9 +11,13 @@ if TYPE_CHECKING: import torch + + from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption else: torch = LazyLoader("torch", globals(), "torch") + ChatTemplateContentFormatOption = object + logger = init_logger(__name__) @@ -43,7 +46,7 @@ class ChatParams: chat_template: str | None = None """The chat template to apply.""" - chat_template_content_format: ChatTemplateContentFormatOption = "auto" + chat_template_content_format: "ChatTemplateContentFormatOption" = "auto" """The format of the chat template.""" chat_template_kwargs: dict[str, Any] = field(default_factory=dict) @@ -163,10 +166,7 @@ def __post_init__(self) -> None: value=truncate_prompt_tokens, ) - def with_kwargs(self, tokenization_kwargs: dict[str, Any] | None): - if tokenization_kwargs is None: - tokenization_kwargs = {} - + def with_kwargs(self, **tokenization_kwargs: Any): max_length = tokenization_kwargs.pop("max_length", self.max_input_tokens) pad_prompt_tokens = tokenization_kwargs.pop( "pad_prompt_tokens", self.pad_prompt_tokens diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index fe2bc327c9f8..36b0bfc662b8 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -19,8 +19,8 @@ WeightTransferUpdateRequest, ) from vllm.engine.arg_utils import AsyncEngineArgs -from vllm.engine.protocol import EngineClient -from vllm.inputs import PromptType, StreamingInput +from vllm.engine.protocol import EngineClient, StreamingInput +from vllm.inputs import PromptType from vllm.logger import init_logger from vllm.lora.request import LoRARequest from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry @@ -272,7 +272,7 @@ def shutdown(self): engine_core.shutdown() if input_processor := getattr(self, "input_processor", None): - input_processor.close() + input_processor.shutdown() handler = getattr(self, "output_handler", None) if handler is not None: diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 1bda736fe6a4..0dfbeb459f7d 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -33,6 +33,7 @@ from vllm.tasks import POOLING_TASKS, SupportedTask from vllm.tokenizers import TokenizerLike from vllm.utils import length_from_prompt_token_ids_or_embeds, random_uuid +from vllm.utils.jsontree import json_iter_leaves from vllm.utils.torch_utils import set_default_torch_num_threads from vllm.v1.engine import EngineCoreRequest from vllm.v1.metrics.stats import MultiModalCacheStats @@ -60,8 +61,6 @@ def __init__( self.generation_config_fields = model_config.try_get_generation_config() self.renderer = renderer or renderer_from_config(vllm_config) - self.mm_registry = mm_registry - self.mm_processor_cache = mm_registry.processor_cache_from_config(vllm_config) self.supports_mm_inputs = mm_registry.supports_multimodal_inputs(model_config) self.mm_encoder_cache_size = 0 @@ -78,7 +77,6 @@ def __init__( vllm_config, renderer=renderer, mm_registry=mm_registry, - mm_processor_cache=self.mm_processor_cache, ) @property @@ -136,7 +134,7 @@ def _validate_params( ) def _parse_mm_items(self, mm_data: MultiModalDataDict) -> MultiModalDataItems: - mm_processor = self.input_preprocessor._get_mm_processor() + mm_processor = self.renderer.get_mm_processor() return mm_processor.info.parse_mm_data(mm_data) def _validate_singleton_mm_uuids(self, prompt: SingletonPrompt) -> None: @@ -415,6 +413,15 @@ def process_inputs( decoder_mm_positions = decoder_inputs["mm_placeholders"] decoder_mm_hashes = decoder_inputs["mm_hashes"] + if not all( + isinstance(leaf, str) for leaf in json_iter_leaves(decoder_mm_hashes) + ): + raise ValueError( + f"mm_hashes must contain only strings, got: {decoder_mm_hashes}. " + "This is likely due to an incorrect custom implementation of " + "MultiModalProcessor.apply method." + ) + # Merge and flatten multimodal placeholders, hashes and inputs # from dictionaries to lists, and sort them by each item's position # in the input sequence. @@ -564,11 +571,10 @@ def _validate_model_inputs( self._validate_model_input(decoder_inputs, prompt_type="decoder") def stat_mm_cache(self) -> MultiModalCacheStats | None: - return self.input_preprocessor.stat_mm_cache() + return self.renderer.stat_mm_cache() def clear_mm_cache(self) -> None: - self.input_preprocessor.clear_mm_cache() + self.renderer.clear_mm_cache() - def close(self) -> None: - if self.mm_processor_cache is not None: - self.mm_processor_cache.close() + def shutdown(self) -> None: + self.renderer.shutdown() diff --git a/vllm/v1/metrics/stats.py b/vllm/v1/metrics/stats.py index 1b7ee105ebf2..4a1e8b6f35ce 100644 --- a/vllm/v1/metrics/stats.py +++ b/vllm/v1/metrics/stats.py @@ -151,6 +151,12 @@ class MultiModalCacheStats(BaseCacheStats): that were queried. """ + def record(self, num_queries: int, num_hits: int) -> None: + """Aggregate request information into the stats.""" + self.requests += 1 + self.queries += num_queries + self.hits += num_hits + @dataclass class KVCacheEvictionEvent: From 45f998c594014037054909029dd013bb912dab98 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 09:35:40 +0000 Subject: [PATCH 02/23] One line Signed-off-by: DarkLight1337 --- vllm/model_executor/models/clip.py | 8 +------- vllm/model_executor/models/lfm2_vl.py | 8 +------- vllm/model_executor/models/mllama4.py | 8 +------- vllm/model_executor/models/nano_nemotron_vl.py | 8 +------- vllm/model_executor/models/nemotron_parse.py | 8 +------- vllm/model_executor/models/ovis.py | 8 +------- vllm/model_executor/models/ovis2_5.py | 8 +------- vllm/model_executor/models/paligemma.py | 8 +------- vllm/model_executor/models/siglip.py | 8 +------- vllm/model_executor/models/ultravox.py | 8 +------- vllm/model_executor/models/whisper.py | 8 +------- 11 files changed, 11 insertions(+), 77 deletions(-) diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index f27a083c224b..3b654b0fd43a 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -128,13 +128,7 @@ def get_hf_processor(self, **kwargs: object): return self.ctx.get_hf_processor(CLIPProcessor, **kwargs) def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": 1} diff --git a/vllm/model_executor/models/lfm2_vl.py b/vllm/model_executor/models/lfm2_vl.py index 1b282fc4dd6e..3355e4016554 100644 --- a/vllm/model_executor/models/lfm2_vl.py +++ b/vllm/model_executor/models/lfm2_vl.py @@ -92,13 +92,7 @@ def get_image_processor(self, **kwargs: object) -> Lfm2VlImageProcessorFast: return self.get_hf_processor(**kwargs).image_processor def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": None} diff --git a/vllm/model_executor/models/mllama4.py b/vllm/model_executor/models/mllama4.py index 6885f02774a0..6b3ca695ac62 100644 --- a/vllm/model_executor/models/mllama4.py +++ b/vllm/model_executor/models/mllama4.py @@ -556,13 +556,7 @@ def get_hf_processor(self, **kwargs: object) -> Llama4Processor: ) def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_supported_mm_limits(self) -> Mapping[str, int | None]: # Although vLLM can support more images from an infra capability diff --git a/vllm/model_executor/models/nano_nemotron_vl.py b/vllm/model_executor/models/nano_nemotron_vl.py index 93d896feb04d..b4c5f6e6439d 100644 --- a/vllm/model_executor/models/nano_nemotron_vl.py +++ b/vllm/model_executor/models/nano_nemotron_vl.py @@ -1095,13 +1095,7 @@ def get_hf_processor( raise NotImplementedError def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": None} diff --git a/vllm/model_executor/models/nemotron_parse.py b/vllm/model_executor/models/nemotron_parse.py index 5be4db56e851..813675a9237f 100644 --- a/vllm/model_executor/models/nemotron_parse.py +++ b/vllm/model_executor/models/nemotron_parse.py @@ -610,13 +610,7 @@ def get_hf_processor(self, **kwargs) -> NemotronParseProcessor: ) def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) @property def skip_prompt_length_check(self) -> bool: diff --git a/vllm/model_executor/models/ovis.py b/vllm/model_executor/models/ovis.py index d7f419cdcaec..990197cc6744 100644 --- a/vllm/model_executor/models/ovis.py +++ b/vllm/model_executor/models/ovis.py @@ -266,13 +266,7 @@ def get_hf_processor(self, **kwargs: object): ) def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_image_segment_len(self) -> int: visual_tokenizer_config = self.get_hf_config().visual_tokenizer_config diff --git a/vllm/model_executor/models/ovis2_5.py b/vllm/model_executor/models/ovis2_5.py index abee3a6bb821..9f2098a95281 100644 --- a/vllm/model_executor/models/ovis2_5.py +++ b/vllm/model_executor/models/ovis2_5.py @@ -185,13 +185,7 @@ def get_hf_processor(self, **kwargs): ) def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_image_processor(self) -> BaseImageProcessor: return self.get_hf_processor().image_processor # type: ignore diff --git a/vllm/model_executor/models/paligemma.py b/vllm/model_executor/models/paligemma.py index d68d4db702cf..0453f6852853 100644 --- a/vllm/model_executor/models/paligemma.py +++ b/vllm/model_executor/models/paligemma.py @@ -104,13 +104,7 @@ def get_vision_encoder_info(self): return get_vision_encoder_info(self.get_hf_config()) def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": 1} diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 5060492769d2..68aa7f34b793 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -113,13 +113,7 @@ def get_hf_processor(self, **kwargs: object): return self.ctx.get_hf_processor(SiglipProcessor, **kwargs) def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": 1} diff --git a/vllm/model_executor/models/ultravox.py b/vllm/model_executor/models/ultravox.py index 156c985f5535..cf8267d2077b 100644 --- a/vllm/model_executor/models/ultravox.py +++ b/vllm/model_executor/models/ultravox.py @@ -135,13 +135,7 @@ def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor: return feature_extractor def get_default_tok_params(self) -> TokenizeParams: - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_data_parser(self): feature_extractor = self.get_feature_extractor() diff --git a/vllm/model_executor/models/whisper.py b/vllm/model_executor/models/whisper.py index 879f332a924c..acc9bcf8fdc1 100644 --- a/vllm/model_executor/models/whisper.py +++ b/vllm/model_executor/models/whisper.py @@ -649,13 +649,7 @@ def get_default_tok_params(self) -> TokenizeParams: # Special tokens should be provided by the user based on the # task and language of their request. Also needed to avoid # appending an EOS token to the prompt which disrupts generation. - return ( - super() - .get_default_tok_params() - .with_kwargs( - add_special_tokens=False, - ) - ) + return super().get_default_tok_params().with_kwargs(add_special_tokens=False) def get_data_parser(self): feature_extractor = self.get_feature_extractor() From 3efd4b81197cb1c5e58bc1566a484c0ad74d8864 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 09:48:13 +0000 Subject: [PATCH 03/23] Fix bad merge Signed-off-by: DarkLight1337 --- vllm/renderers/base.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 95a08fa27f0f..cc2363fa6e0e 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -426,14 +426,6 @@ def render_cmpl( if tok_params is None: tok_params = self.default_cmpl_tok_params - dict_prompts = self.render_prompts(prompts) - - # NOTE: Some MM models have non-default `add_special_tokens` - # so we handle tokenization in multi-modal processor - if self.model_config.is_multimodal_model: - self._apply_prompt_extras(dict_prompts, prompt_extras) - return dict_prompts - dict_prompts = self.render_prompts(prompts) tok_prompts = self.tokenize_prompts(dict_prompts, tok_params) From cb9f5983b9e3a19f8346c4a2e216f11888f41cf6 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 09:58:29 +0000 Subject: [PATCH 04/23] Doc Signed-off-by: DarkLight1337 --- vllm/benchmarks/mm_processor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/vllm/benchmarks/mm_processor.py b/vllm/benchmarks/mm_processor.py index 4d372da89e74..465de07bc571 100644 --- a/vllm/benchmarks/mm_processor.py +++ b/vllm/benchmarks/mm_processor.py @@ -18,7 +18,7 @@ import json import time from datetime import datetime -from typing import Any +from typing import TYPE_CHECKING, Any import numpy as np @@ -30,13 +30,17 @@ from vllm.engine.arg_utils import EngineArgs from vllm.utils.gc_utils import freeze_gc_heap from vllm.utils.import_utils import PlaceholderModule -from vllm.v1.engine.llm_engine import LLMEngine try: import pandas as pd except ImportError: pd = PlaceholderModule("pandas") +if TYPE_CHECKING: # Avoid having to mock during docs build + from vllm.v1.engine.llm_engine import LLMEngine +else: + LLMEngine = object + def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, float]]: """ From 40865fc2a4cb363c56b6e9ae70e8dc204def851d Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 10:10:29 +0000 Subject: [PATCH 05/23] Simplify Signed-off-by: DarkLight1337 --- vllm/entrypoints/llm.py | 2 +- vllm/v1/engine/async_llm.py | 12 ++++++------ vllm/v1/engine/input_processor.py | 10 ---------- vllm/v1/engine/llm_engine.py | 4 ++-- 4 files changed, 9 insertions(+), 19 deletions(-) diff --git a/vllm/entrypoints/llm.py b/vllm/entrypoints/llm.py index f3c4d7540c5d..d27fa7074c35 100644 --- a/vllm/entrypoints/llm.py +++ b/vllm/entrypoints/llm.py @@ -384,7 +384,7 @@ def get_world_size(self, include_dp: bool = True) -> int: return parallel_config.world_size def reset_mm_cache(self) -> None: - self.input_processor.clear_mm_cache() + self.renderer.clear_mm_cache() self.llm_engine.reset_mm_cache() def get_default_sampling_params(self) -> SamplingParams: diff --git a/vllm/v1/engine/async_llm.py b/vllm/v1/engine/async_llm.py index 36b0bfc662b8..9f92dbe97f1c 100644 --- a/vllm/v1/engine/async_llm.py +++ b/vllm/v1/engine/async_llm.py @@ -268,12 +268,12 @@ def shutdown(self): shutdown_prometheus() + if renderer := getattr(self, "renderer", None): + renderer.shutdown() + if engine_core := getattr(self, "engine_core", None): engine_core.shutdown() - if input_processor := getattr(self, "input_processor", None): - input_processor.shutdown() - handler = getattr(self, "output_handler", None) if handler is not None: cancel_task_threadsafe(handler) @@ -654,7 +654,7 @@ def _run_output_handler(self): output_processor = self.output_processor log_stats = self.log_stats logger_manager = self.logger_manager - input_processor = self.input_processor + renderer = self.renderer chunk_size = envs.VLLM_V1_OUTPUT_PROC_CHUNK_SIZE async def output_handler(): @@ -702,7 +702,7 @@ async def output_handler(): engine_idx=outputs.engine_index, scheduler_stats=outputs.scheduler_stats, iteration_stats=iteration_stats, - mm_cache_stats=input_processor.stat_mm_cache(), + mm_cache_stats=renderer.stat_mm_cache(), ) except Exception as e: logger.exception("AsyncLLM output_handler failed.") @@ -881,7 +881,7 @@ async def stop_profile(self) -> None: await asyncio.gather(*coros) async def reset_mm_cache(self) -> None: - self.input_processor.clear_mm_cache() + self.renderer.clear_mm_cache() await self.engine_core.reset_mm_cache_async() async def reset_prefix_cache( diff --git a/vllm/v1/engine/input_processor.py b/vllm/v1/engine/input_processor.py index 0dfbeb459f7d..c51adf85470f 100644 --- a/vllm/v1/engine/input_processor.py +++ b/vllm/v1/engine/input_processor.py @@ -36,7 +36,6 @@ from vllm.utils.jsontree import json_iter_leaves from vllm.utils.torch_utils import set_default_torch_num_threads from vllm.v1.engine import EngineCoreRequest -from vllm.v1.metrics.stats import MultiModalCacheStats logger = init_logger(__name__) @@ -569,12 +568,3 @@ def _validate_model_inputs( self._validate_model_input(encoder_inputs, prompt_type="encoder") self._validate_model_input(decoder_inputs, prompt_type="decoder") - - def stat_mm_cache(self) -> MultiModalCacheStats | None: - return self.renderer.stat_mm_cache() - - def clear_mm_cache(self) -> None: - self.renderer.clear_mm_cache() - - def shutdown(self) -> None: - self.renderer.shutdown() diff --git a/vllm/v1/engine/llm_engine.py b/vllm/v1/engine/llm_engine.py index c7eb93dc8ad7..851c0604bf98 100644 --- a/vllm/v1/engine/llm_engine.py +++ b/vllm/v1/engine/llm_engine.py @@ -320,7 +320,7 @@ def step(self) -> list[RequestOutput | PoolingRequestOutput]: self.logger_manager.record( scheduler_stats=outputs.scheduler_stats, iteration_stats=iteration_stats, - mm_cache_stats=self.input_processor.stat_mm_cache(), + mm_cache_stats=self.renderer.stat_mm_cache(), ) self.do_log_stats_with_interval() @@ -333,7 +333,7 @@ def stop_profile(self): self.engine_core.profile(False) def reset_mm_cache(self): - self.input_processor.clear_mm_cache() + self.renderer.clear_mm_cache() self.engine_core.reset_mm_cache() def reset_prefix_cache( From ba4401b26c50d08a4ea593f0bd1ece477d8587f7 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 10:10:52 +0000 Subject: [PATCH 06/23] Fix Signed-off-by: DarkLight1337 --- vllm/benchmarks/mm_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vllm/benchmarks/mm_processor.py b/vllm/benchmarks/mm_processor.py index 465de07bc571..6d5a6d95a547 100644 --- a/vllm/benchmarks/mm_processor.py +++ b/vllm/benchmarks/mm_processor.py @@ -50,7 +50,7 @@ def get_timing_stats_from_engine(llm_engine: LLMEngine) -> dict[str, dict[str, f prompt update) and encoder forward pass timing, merged by request_id. Args: - engine: The LLM engine (has input_processor and workers). + llm_engine: The LLM engine (has input_processor and workers). Returns: Dictionary mapping request_id to merged stats dict containing From bdaf7aff84e67e95178de72b7894bb246b93c1c3 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 11:41:09 +0000 Subject: [PATCH 07/23] Fix Signed-off-by: DarkLight1337 --- vllm/renderers/base.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index cc2363fa6e0e..19193e5f3d9e 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -158,7 +158,7 @@ def get_eos_token_id(self) -> int | None: @cached_property def default_cmpl_tok_params(self) -> TokenizeParams: - mm_processor = self.get_mm_processor() + mm_processor = self.mm_processor if mm_processor is not None: return mm_processor.info.default_tok_params @@ -173,7 +173,7 @@ def default_cmpl_tok_params(self) -> TokenizeParams: @cached_property def default_chat_tok_params(self) -> TokenizeParams: - mm_processor = self.get_mm_processor() + mm_processor = self.mm_processor if mm_processor is not None: return mm_processor.info.default_tok_params From b6ff227f8723ee14e1c7902f76200ba8f13c5533 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 12:31:16 +0000 Subject: [PATCH 08/23] Fix Signed-off-by: DarkLight1337 --- vllm/renderers/base.py | 52 +++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 31 deletions(-) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 19193e5f3d9e..8319aaaa0333 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -28,7 +28,6 @@ ChatCompletionMessageParam, ConversationMessage, ) - from vllm.multimodal.cache import BaseMultiModalProcessorCache from vllm.multimodal.processing import BaseMultiModalProcessor logger = init_logger(__name__) @@ -53,9 +52,23 @@ def __init__(self, config: "VllmConfig") -> None: # Lazy initialization since offline LLM doesn't use async self._async_tokenizer: AsyncMicrobatchTokenizer | None = None - # Lazy initialization since it's only for MM models self._mm_processor: BaseMultiModalProcessor | None = None + self._mm_processor_cache: BaseMultiModalProcessor | None = None self._mm_cache_stats: MultiModalCacheStats | None = None + if config.model_config.is_multimodal_model: + from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry + + mm_processor_cache = mm_registry.processor_cache_from_config(config) + self._mm_processor = mm_registry.create_processor( + config.model_config, + config.observability_config, + tokenizer=self.tokenizer, + cache=mm_processor_cache, + ) + self._mm_processor_cache = mm_processor_cache + self._mm_cache_stats = ( + MultiModalCacheStats() if mm_processor_cache else None + ) @property @abstractmethod @@ -75,39 +88,16 @@ def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer: return self._async_tokenizer - @cached_property + @property def mm_processor(self) -> "BaseMultiModalProcessor | None": - vllm_config = self.config - if not vllm_config.model_config.is_multimodal_model: - return None - - return self.get_mm_processor() + return self._mm_processor def get_mm_processor(self) -> "BaseMultiModalProcessor": if self._mm_processor is None: - from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry - - vllm_config = self.config - mm_processor_cache = mm_registry.processor_cache_from_config(vllm_config) - self._mm_processor = mm_registry.create_processor( - vllm_config.model_config, - vllm_config.observability_config, - tokenizer=self.tokenizer, - cache=mm_processor_cache, - ) - self._mm_cache_stats = ( - MultiModalCacheStats() if mm_processor_cache else None - ) + raise ValueError("Multi-modal processor not available for text-only models") return self._mm_processor - def get_mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None": - mm_processor = self.mm_processor - if mm_processor is None: - return None - - return mm_processor.cache - def stat_mm_cache(self) -> MultiModalCacheStats | None: mm_cache_stats = self._mm_cache_stats if mm_cache_stats is None: @@ -118,7 +108,7 @@ def stat_mm_cache(self) -> MultiModalCacheStats | None: return mm_cache_stats def update_mm_cache_stats(self) -> None: - mm_processor_cache = self.get_mm_processor_cache() + mm_processor_cache = self._mm_processor_cache mm_cache_stats = self._mm_cache_stats if mm_processor_cache and mm_cache_stats: @@ -126,7 +116,7 @@ def update_mm_cache_stats(self) -> None: mm_cache_stats.record(delta.total, delta.hits) def clear_mm_cache(self) -> None: - mm_processor_cache = self.get_mm_processor_cache() + mm_processor_cache = self._mm_processor_cache if mm_processor_cache is not None: mm_processor_cache.clear_cache() @@ -134,7 +124,7 @@ def clear_mm_cache(self) -> None: self._mm_cache_stats.reset = True def shutdown(self) -> None: - mm_processor_cache = self.get_mm_processor_cache() + mm_processor_cache = self._mm_processor_cache if mm_processor_cache is not None: mm_processor_cache.close() From 44121510e3d428b3c004923639d1b94e46da9a2f Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 12:31:50 +0000 Subject: [PATCH 09/23] Fix Signed-off-by: DarkLight1337 --- vllm/renderers/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 8319aaaa0333..34d6818c2ecc 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -28,6 +28,7 @@ ChatCompletionMessageParam, ConversationMessage, ) + from vllm.multimodal.cache import BaseMultiModalProcessorCache from vllm.multimodal.processing import BaseMultiModalProcessor logger = init_logger(__name__) @@ -53,7 +54,7 @@ def __init__(self, config: "VllmConfig") -> None: self._async_tokenizer: AsyncMicrobatchTokenizer | None = None self._mm_processor: BaseMultiModalProcessor | None = None - self._mm_processor_cache: BaseMultiModalProcessor | None = None + self._mm_processor_cache: BaseMultiModalProcessorCache | None = None self._mm_cache_stats: MultiModalCacheStats | None = None if config.model_config.is_multimodal_model: from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry From 123cf8e191c01e19ca6f123db5c7f5dc59b66aa9 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 13:20:34 +0000 Subject: [PATCH 10/23] Fix init order Signed-off-by: DarkLight1337 --- vllm/renderers/base.py | 26 ++++++++++++-------------- vllm/renderers/deepseek_v32.py | 20 +++----------------- vllm/renderers/grok2.py | 19 +++---------------- vllm/renderers/hf.py | 25 ++++++------------------- vllm/renderers/mistral.py | 19 +++---------------- 5 files changed, 27 insertions(+), 82 deletions(-) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 34d6818c2ecc..2af0fc8756c8 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -4,7 +4,9 @@ from abc import ABC, abstractmethod from collections.abc import Sequence from functools import cached_property -from typing import TYPE_CHECKING, Any, overload +from typing import TYPE_CHECKING, Any, Generic, overload + +from typing_extensions import TypeVar from vllm.inputs import EmbedsPrompt, TextPrompt, TokensPrompt from vllm.logger import init_logger @@ -34,7 +36,10 @@ logger = init_logger(__name__) -class BaseRenderer(ABC): +_T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike) + + +class BaseRenderer(ABC, Generic[_T]): @classmethod @abstractmethod def from_config( @@ -44,16 +49,18 @@ def from_config( ) -> "BaseRenderer": raise NotImplementedError - def __init__(self, config: "VllmConfig") -> None: + def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None: super().__init__() self.config = config self.model_config = config.model_config + self.tokenizer = tokenizer + # Lazy initialization since offline LLM doesn't use async self._async_tokenizer: AsyncMicrobatchTokenizer | None = None - self._mm_processor: BaseMultiModalProcessor | None = None + self.mm_processor: BaseMultiModalProcessor | None = None self._mm_processor_cache: BaseMultiModalProcessorCache | None = None self._mm_cache_stats: MultiModalCacheStats | None = None if config.model_config.is_multimodal_model: @@ -71,12 +78,7 @@ def __init__(self, config: "VllmConfig") -> None: MultiModalCacheStats() if mm_processor_cache else None ) - @property - @abstractmethod - def tokenizer(self) -> TokenizerLike | None: - raise NotImplementedError - - def get_tokenizer(self) -> TokenizerLike: + def get_tokenizer(self) -> _T: tokenizer = self.tokenizer if tokenizer is None: raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") @@ -89,10 +91,6 @@ def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer: return self._async_tokenizer - @property - def mm_processor(self) -> "BaseMultiModalProcessor | None": - return self._mm_processor - def get_mm_processor(self) -> "BaseMultiModalProcessor": if self._mm_processor is None: raise ValueError("Multi-modal processor not available for text-only models") diff --git a/vllm/renderers/deepseek_v32.py b/vllm/renderers/deepseek_v32.py index f03a5973f3f7..092a12b686d2 100644 --- a/vllm/renderers/deepseek_v32.py +++ b/vllm/renderers/deepseek_v32.py @@ -13,7 +13,6 @@ from vllm.tokenizers import cached_get_tokenizer from vllm.tokenizers.deepseek_v32 import DeepseekV32Tokenizer -from ..tokenizers.hf import HfTokenizer from .base import BaseRenderer from .inputs import DictPrompt from .inputs.preprocess import parse_dec_only_prompt @@ -22,13 +21,13 @@ logger = init_logger(__name__) -class DeepseekV32Renderer(BaseRenderer): +class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]): @classmethod def from_config( cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], - ) -> "BaseRenderer": + ) -> "DeepseekV32Renderer": return cls(config, tokenizer_kwargs) def __init__( @@ -36,8 +35,6 @@ def __init__( config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> None: - super().__init__(config) - model_config = self.model_config if model_config.skip_tokenizer_init: tokenizer = None @@ -47,18 +44,7 @@ def __init__( **tokenizer_kwargs, ) - self._tokenizer = tokenizer - - @property - def tokenizer(self) -> HfTokenizer | None: - return self._tokenizer - - def get_tokenizer(self) -> HfTokenizer: - tokenizer = self.tokenizer - if tokenizer is None: - raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") - - return tokenizer + super().__init__(config, tokenizer) def render_messages( self, diff --git a/vllm/renderers/grok2.py b/vllm/renderers/grok2.py index 7e8681d82364..7fcf0b4b23b3 100644 --- a/vllm/renderers/grok2.py +++ b/vllm/renderers/grok2.py @@ -21,13 +21,13 @@ logger = init_logger(__name__) -class Grok2Renderer(BaseRenderer): +class Grok2Renderer(BaseRenderer[Grok2Tokenizer]): @classmethod def from_config( cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], - ) -> "BaseRenderer": + ) -> "Grok2Renderer": return cls(config, tokenizer_kwargs) def __init__( @@ -35,8 +35,6 @@ def __init__( config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> None: - super().__init__(config) - model_config = self.model_config if model_config.skip_tokenizer_init: tokenizer = None @@ -46,18 +44,7 @@ def __init__( **tokenizer_kwargs, ) - self._tokenizer = tokenizer - - @property - def tokenizer(self) -> Grok2Tokenizer | None: - return self._tokenizer - - def get_tokenizer(self) -> Grok2Tokenizer: - tokenizer = self.tokenizer - if tokenizer is None: - raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") - - return tokenizer + super().__init__(config, tokenizer) def render_messages( self, diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 407b28ae18d9..c36ef92b9cf4 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -585,13 +585,13 @@ def replace_vision_chunk_video_placeholder( return prompt_raw -class HfRenderer(BaseRenderer): +class HfRenderer(BaseRenderer[CachedHfTokenizer]): @classmethod def from_config( cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], - ) -> "BaseRenderer": + ) -> "HfRenderer": return cls(config, tokenizer_kwargs) def __init__( @@ -599,13 +599,7 @@ def __init__( config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> None: - super().__init__(config) - model_config = self.model_config - self.use_unified_vision_chunk = getattr( - model_config.hf_config, "use_unified_vision_chunk", False - ) - if model_config.skip_tokenizer_init: tokenizer = None else: @@ -617,18 +611,11 @@ def __init__( ), ) - self._tokenizer = tokenizer + super().__init__(config, tokenizer) - @property - def tokenizer(self) -> HfTokenizer | None: - return self._tokenizer - - def get_tokenizer(self) -> HfTokenizer: - tokenizer = self.tokenizer - if tokenizer is None: - raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") - - return tokenizer + self.use_unified_vision_chunk = getattr( + model_config.hf_config, "use_unified_vision_chunk", False + ) def render_messages( self, diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index ae8078f41daf..03c238e681f6 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -50,13 +50,13 @@ def safe_apply_chat_template( raise ValueError(str(e)) from e -class MistralRenderer(BaseRenderer): +class MistralRenderer(BaseRenderer[MistralTokenizer]): @classmethod def from_config( cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], - ) -> "BaseRenderer": + ) -> "MistralRenderer": return cls(config, tokenizer_kwargs) def __init__( @@ -64,8 +64,6 @@ def __init__( config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> None: - super().__init__(config) - model_config = self.model_config if model_config.skip_tokenizer_init: tokenizer = None @@ -75,24 +73,13 @@ def __init__( **tokenizer_kwargs, ) - self._tokenizer = tokenizer + super().__init__(config, tokenizer) self._apply_chat_template_executor = ThreadPoolExecutor(max_workers=1) self._apply_chat_template_async = make_async( safe_apply_chat_template, executor=self._apply_chat_template_executor ) - @property - def tokenizer(self) -> MistralTokenizer | None: - return self._tokenizer - - def get_tokenizer(self) -> MistralTokenizer: - tokenizer = self.tokenizer - if tokenizer is None: - raise ValueError("Tokenizer not available when `skip_tokenizer_init=True`") - - return tokenizer - def render_messages( self, messages: list[ChatCompletionMessageParam], From d00556854021067c6770d7a4672480cb6b0a4310 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 13:39:02 +0000 Subject: [PATCH 11/23] Simplify Signed-off-by: DarkLight1337 --- vllm/renderers/deepseek_v32.py | 11 ++--------- vllm/renderers/grok2.py | 11 ++--------- vllm/renderers/hf.py | 20 ++++++++++---------- vllm/renderers/mistral.py | 16 ++++++++-------- vllm/renderers/terratorch.py | 15 ++------------- 5 files changed, 24 insertions(+), 49 deletions(-) diff --git a/vllm/renderers/deepseek_v32.py b/vllm/renderers/deepseek_v32.py index 092a12b686d2..91416b968085 100644 --- a/vllm/renderers/deepseek_v32.py +++ b/vllm/renderers/deepseek_v32.py @@ -28,14 +28,7 @@ def from_config( config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> "DeepseekV32Renderer": - return cls(config, tokenizer_kwargs) - - def __init__( - self, - config: VllmConfig, - tokenizer_kwargs: dict[str, Any], - ) -> None: - model_config = self.model_config + model_config = config.model_config if model_config.skip_tokenizer_init: tokenizer = None else: @@ -44,7 +37,7 @@ def __init__( **tokenizer_kwargs, ) - super().__init__(config, tokenizer) + return cls(config, tokenizer) def render_messages( self, diff --git a/vllm/renderers/grok2.py b/vllm/renderers/grok2.py index 7fcf0b4b23b3..433c0c93a801 100644 --- a/vllm/renderers/grok2.py +++ b/vllm/renderers/grok2.py @@ -28,14 +28,7 @@ def from_config( config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> "Grok2Renderer": - return cls(config, tokenizer_kwargs) - - def __init__( - self, - config: VllmConfig, - tokenizer_kwargs: dict[str, Any], - ) -> None: - model_config = self.model_config + model_config = config.model_config if model_config.skip_tokenizer_init: tokenizer = None else: @@ -44,7 +37,7 @@ def __init__( **tokenizer_kwargs, ) - super().__init__(config, tokenizer) + return cls(config, tokenizer) def render_messages( self, diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index c36ef92b9cf4..8b03908e03b5 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -585,21 +585,14 @@ def replace_vision_chunk_video_placeholder( return prompt_raw -class HfRenderer(BaseRenderer[CachedHfTokenizer]): +class HfRenderer(BaseRenderer[HfTokenizer]): @classmethod def from_config( cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> "HfRenderer": - return cls(config, tokenizer_kwargs) - - def __init__( - self, - config: VllmConfig, - tokenizer_kwargs: dict[str, Any], - ) -> None: - model_config = self.model_config + model_config = config.model_config if model_config.skip_tokenizer_init: tokenizer = None else: @@ -611,10 +604,17 @@ def __init__( ), ) + return cls(config, tokenizer) + + def __init__( + self, + config: VllmConfig, + tokenizer: HfTokenizer | None, + ) -> None: super().__init__(config, tokenizer) self.use_unified_vision_chunk = getattr( - model_config.hf_config, "use_unified_vision_chunk", False + config.model_config.hf_config, "use_unified_vision_chunk", False ) def render_messages( diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index 03c238e681f6..aa1b809e389c 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -57,14 +57,7 @@ def from_config( config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> "MistralRenderer": - return cls(config, tokenizer_kwargs) - - def __init__( - self, - config: VllmConfig, - tokenizer_kwargs: dict[str, Any], - ) -> None: - model_config = self.model_config + model_config = config.model_config if model_config.skip_tokenizer_init: tokenizer = None else: @@ -73,6 +66,13 @@ def __init__( **tokenizer_kwargs, ) + return cls(config, tokenizer) + + def __init__( + self, + config: VllmConfig, + tokenizer: MistralTokenizer | None, + ) -> None: super().__init__(config, tokenizer) self._apply_chat_template_executor = ThreadPoolExecutor(max_workers=1) diff --git a/vllm/renderers/terratorch.py b/vllm/renderers/terratorch.py index 0ee97f852f42..28c272a663fc 100644 --- a/vllm/renderers/terratorch.py +++ b/vllm/renderers/terratorch.py @@ -10,7 +10,6 @@ parse_chat_messages_async, ) from vllm.logger import init_logger -from vllm.tokenizers import TokenizerLike from .base import BaseRenderer from .inputs import DictPrompt @@ -27,21 +26,11 @@ def from_config( config: VllmConfig, tokenizer_kwargs: dict[str, Any], ) -> "BaseRenderer": - return cls(config) - - def __init__(self, config: VllmConfig) -> None: - super().__init__(config) - - model_config = self.model_config + model_config = config.model_config if not model_config.skip_tokenizer_init: raise ValueError("Terratorch renderer requires `skip_tokenizer_init=True`") - @property - def tokenizer(self) -> TokenizerLike | None: - return None - - def get_tokenizer(self) -> TokenizerLike: - raise ValueError("Tokenizer not available for Terratorch renderer") + return cls(config, None) def render_messages( self, From 1c645b64e2f8bd4a23ec73dd1d7a52e50c3ac6be Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 13:40:51 +0000 Subject: [PATCH 12/23] mypy Signed-off-by: DarkLight1337 --- vllm/renderers/deepseek_v32.py | 2 +- vllm/renderers/grok2.py | 2 +- vllm/renderers/hf.py | 2 +- vllm/renderers/mistral.py | 2 +- vllm/renderers/terratorch.py | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/vllm/renderers/deepseek_v32.py b/vllm/renderers/deepseek_v32.py index 91416b968085..67cee8752d99 100644 --- a/vllm/renderers/deepseek_v32.py +++ b/vllm/renderers/deepseek_v32.py @@ -23,7 +23,7 @@ class DeepseekV32Renderer(BaseRenderer[DeepseekV32Tokenizer]): @classmethod - def from_config( + def from_config( # type: ignore[override] cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], diff --git a/vllm/renderers/grok2.py b/vllm/renderers/grok2.py index 433c0c93a801..bc365cb7c851 100644 --- a/vllm/renderers/grok2.py +++ b/vllm/renderers/grok2.py @@ -23,7 +23,7 @@ class Grok2Renderer(BaseRenderer[Grok2Tokenizer]): @classmethod - def from_config( + def from_config( # type: ignore[override] cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 8b03908e03b5..a2c281b9d0fa 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -587,7 +587,7 @@ def replace_vision_chunk_video_placeholder( class HfRenderer(BaseRenderer[HfTokenizer]): @classmethod - def from_config( + def from_config( # type: ignore[override] cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], diff --git a/vllm/renderers/mistral.py b/vllm/renderers/mistral.py index aa1b809e389c..feea19fba275 100644 --- a/vllm/renderers/mistral.py +++ b/vllm/renderers/mistral.py @@ -52,7 +52,7 @@ def safe_apply_chat_template( class MistralRenderer(BaseRenderer[MistralTokenizer]): @classmethod - def from_config( + def from_config( # type: ignore[override] cls, config: VllmConfig, tokenizer_kwargs: dict[str, Any], diff --git a/vllm/renderers/terratorch.py b/vllm/renderers/terratorch.py index 28c272a663fc..3e9f1ce69834 100644 --- a/vllm/renderers/terratorch.py +++ b/vllm/renderers/terratorch.py @@ -23,9 +23,9 @@ class TerratorchRenderer(BaseRenderer): @classmethod def from_config( cls, - config: VllmConfig, + config: VllmConfig, # type: ignore[override] tokenizer_kwargs: dict[str, Any], - ) -> "BaseRenderer": + ) -> "TerratorchRenderer": model_config = config.model_config if not model_config.skip_tokenizer_init: raise ValueError("Terratorch renderer requires `skip_tokenizer_init=True`") From caa68e097b7d10e2b31d997724417a7f4d085b23 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 15:49:13 +0000 Subject: [PATCH 13/23] Fix init Signed-off-by: DarkLight1337 --- tests/entrypoints/openai/test_chat_error.py | 3 ++- tests/entrypoints/openai/test_completion_error.py | 3 ++- tests/entrypoints/openai/test_lora_resolvers.py | 3 ++- tests/entrypoints/openai/test_serving_chat.py | 7 ++++--- tests/renderers/test_completions.py | 3 ++- tests/renderers/test_mistral.py | 3 ++- 6 files changed, 14 insertions(+), 8 deletions(-) diff --git a/tests/entrypoints/openai/test_chat_error.py b/tests/entrypoints/openai/test_chat_error.py index 6095d1ec83c5..41b8b52c42ce 100644 --- a/tests/entrypoints/openai/test_chat_error.py +++ b/tests/entrypoints/openai/test_chat_error.py @@ -54,6 +54,7 @@ class MockModelConfig: media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict) skip_tokenizer_init = False is_encoder_decoder: bool = False + is_multimodal_model: bool = False def get_diff_sampling_param(self): return self.diff_sampling_param or {} @@ -67,7 +68,7 @@ class MockVllmConfig: def _build_renderer(model_config: MockModelConfig): _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config) - return HfRenderer( + return HfRenderer.from_config( MockVllmConfig(model_config), tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name}, ) diff --git a/tests/entrypoints/openai/test_completion_error.py b/tests/entrypoints/openai/test_completion_error.py index d5a266831583..a7f6a75e0e72 100644 --- a/tests/entrypoints/openai/test_completion_error.py +++ b/tests/entrypoints/openai/test_completion_error.py @@ -53,6 +53,7 @@ class MockModelConfig: media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict) skip_tokenizer_init = False is_encoder_decoder: bool = False + is_multimodal_model: bool = False def get_diff_sampling_param(self): return self.diff_sampling_param or {} @@ -78,7 +79,7 @@ def _build_serving_completion(engine: AsyncLLM) -> OpenAIServingCompletion: def _build_renderer(model_config: MockModelConfig): _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config) - return HfRenderer( + return HfRenderer.from_config( MockVllmConfig(model_config), tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name}, ) diff --git a/tests/entrypoints/openai/test_lora_resolvers.py b/tests/entrypoints/openai/test_lora_resolvers.py index 450a788a390b..0988ff64486b 100644 --- a/tests/entrypoints/openai/test_lora_resolvers.py +++ b/tests/entrypoints/openai/test_lora_resolvers.py @@ -52,6 +52,7 @@ class MockModelConfig: generation_config: str = "auto" skip_tokenizer_init: bool = False is_encoder_decoder: bool = False + is_multimodal_model: bool = False def get_diff_sampling_param(self): return self.diff_sampling_param or {} @@ -95,7 +96,7 @@ def register_mock_resolver(): def _build_renderer(model_config: MockModelConfig): _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config) - return HfRenderer( + return HfRenderer.from_config( MockVllmConfig(model_config), tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name}, ) diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 2cef772c293e..0a3704a27d87 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -529,6 +529,7 @@ class MockModelConfig: media_io_kwargs: dict[str, dict[str, Any]] = field(default_factory=dict) skip_tokenizer_init: bool = False is_encoder_decoder: bool = False + is_multimodal_model: bool = False def get_diff_sampling_param(self): return self.diff_sampling_param or {} @@ -542,7 +543,7 @@ class MockVllmConfig: def _build_renderer(model_config: MockModelConfig): _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config) - return HfRenderer( + return HfRenderer.from_config( MockVllmConfig(model_config), tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name}, ) @@ -754,7 +755,7 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated(): mock_engine.io_processor = MagicMock() mock_tokenizer = MagicMock(spec=MistralTokenizer) - mock_renderer = MistralRenderer( + mock_renderer = MistralRenderer.from_config( MockVllmConfig(mock_engine.model_config), tokenizer_kwargs={}, ) @@ -796,7 +797,7 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected(): mock_engine.io_processor = MagicMock() mock_tokenizer = MagicMock(spec=MistralTokenizer) - mock_renderer = MistralRenderer( + mock_renderer = MistralRenderer.from_config( MockVllmConfig(mock_engine.model_config), tokenizer_kwargs={}, ) diff --git a/tests/renderers/test_completions.py b/tests/renderers/test_completions.py index ec6d8a688009..69b349250ffa 100644 --- a/tests/renderers/test_completions.py +++ b/tests/renderers/test_completions.py @@ -38,6 +38,7 @@ class MockModelConfig: enable_prompt_embeds: bool = True skip_tokenizer_init: bool = False is_encoder_decoder: bool = False + is_multimodal_model: bool = False @dataclass @@ -76,7 +77,7 @@ def _build_renderer( ): _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config) - renderer = HfRenderer( + renderer = HfRenderer.from_config( MockVllmConfig(model_config), tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name}, ) diff --git a/tests/renderers/test_mistral.py b/tests/renderers/test_mistral.py index 8c68f750a700..edc0ecf395d2 100644 --- a/tests/renderers/test_mistral.py +++ b/tests/renderers/test_mistral.py @@ -36,6 +36,7 @@ class MockModelConfig: enable_prompt_embeds: bool = True skip_tokenizer_init: bool = False is_encoder_decoder: bool = False + is_multimodal_model: bool = False @dataclass @@ -55,7 +56,7 @@ def mocked_apply_chat_template(*_args, **_kwargs): mock_model_config = MockModelConfig(skip_tokenizer_init=True) mock_tokenizer = Mock(spec=MistralTokenizer) mock_tokenizer.apply_chat_template = mocked_apply_chat_template - mock_renderer = MistralRenderer( + mock_renderer = MistralRenderer.from_config( MockVllmConfig(mock_model_config), tokenizer_kwargs={}, ) From 9f22e6eba8be5bd5978d00e6fa163b20e7ec9e1d Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Fri, 13 Feb 2026 16:23:39 +0000 Subject: [PATCH 14/23] Simplify Signed-off-by: DarkLight1337 --- tests/entrypoints/openai/test_serving_chat.py | 10 ++++---- tests/renderers/test_completions.py | 23 ++++++++++--------- tests/renderers/test_mistral.py | 5 ++-- 3 files changed, 18 insertions(+), 20 deletions(-) diff --git a/tests/entrypoints/openai/test_serving_chat.py b/tests/entrypoints/openai/test_serving_chat.py index 0a3704a27d87..7d0b513aa195 100644 --- a/tests/entrypoints/openai/test_serving_chat.py +++ b/tests/entrypoints/openai/test_serving_chat.py @@ -755,11 +755,10 @@ async def test_serving_chat_mistral_token_ids_prompt_is_validated(): mock_engine.io_processor = MagicMock() mock_tokenizer = MagicMock(spec=MistralTokenizer) - mock_renderer = MistralRenderer.from_config( + mock_renderer = MistralRenderer( MockVllmConfig(mock_engine.model_config), - tokenizer_kwargs={}, + tokenizer=mock_tokenizer, ) - mock_renderer._tokenizer = mock_tokenizer # Force the Mistral chat template renderer to return token IDs. # Choose a prompt length that is < max_model_len, but large enough that # adding max_tokens should exceed the model context window. @@ -797,11 +796,10 @@ async def test_serving_chat_mistral_token_ids_prompt_too_long_is_rejected(): mock_engine.io_processor = MagicMock() mock_tokenizer = MagicMock(spec=MistralTokenizer) - mock_renderer = MistralRenderer.from_config( + mock_renderer = MistralRenderer( MockVllmConfig(mock_engine.model_config), - tokenizer_kwargs={}, + tokenizer=mock_tokenizer, ) - mock_renderer._tokenizer = mock_tokenizer # prompt_token_ids length == max_model_len should be rejected for # completion-like requests (ChatCompletionRequest). mock_renderer.render_messages_async = AsyncMock( diff --git a/tests/renderers/test_completions.py b/tests/renderers/test_completions.py index 69b349250ffa..03e1a655ae43 100644 --- a/tests/renderers/test_completions.py +++ b/tests/renderers/test_completions.py @@ -77,17 +77,18 @@ def _build_renderer( ): _, tokenizer_name, _, kwargs = tokenizer_args_from_config(model_config) - renderer = HfRenderer.from_config( + renderer = HfRenderer( MockVllmConfig(model_config), - tokenizer_kwargs={**kwargs, "tokenizer_name": tokenizer_name}, + tokenizer=( + None + if model_config.skip_tokenizer_init + else DummyTokenizer( + truncation_side=truncation_side, + max_chars_per_token=max_chars_per_token, + ) + ), ) - if not model_config.skip_tokenizer_init: - renderer._tokenizer = DummyTokenizer( - truncation_side=truncation_side, - max_chars_per_token=max_chars_per_token, - ) - return renderer @@ -278,7 +279,7 @@ def test_text_max_length_exceeded_obvious(self): ) # Should not even attempt tokenization - assert renderer._tokenizer._captured_encode_kwargs == {} + assert renderer.tokenizer._captured_encode_kwargs == {} def test_text_max_length_exceeded_nonobvious(self): renderer = _build_renderer(MockModelConfig(), max_chars_per_token=2) @@ -299,8 +300,8 @@ def test_text_max_length_exceeded_nonobvious(self): ) # Should only tokenize the first max_total_tokens + 1 tokens - assert renderer._tokenizer._captured_encode_kwargs["truncation"] is True - assert renderer._tokenizer._captured_encode_kwargs["max_length"] == 101 + assert renderer.tokenizer._captured_encode_kwargs["truncation"] is True + assert renderer.tokenizer._captured_encode_kwargs["max_length"] == 101 def test_token_max_length_exceeded(self): renderer = _build_renderer(MockModelConfig()) diff --git a/tests/renderers/test_mistral.py b/tests/renderers/test_mistral.py index edc0ecf395d2..40235491d8c1 100644 --- a/tests/renderers/test_mistral.py +++ b/tests/renderers/test_mistral.py @@ -56,11 +56,10 @@ def mocked_apply_chat_template(*_args, **_kwargs): mock_model_config = MockModelConfig(skip_tokenizer_init=True) mock_tokenizer = Mock(spec=MistralTokenizer) mock_tokenizer.apply_chat_template = mocked_apply_chat_template - mock_renderer = MistralRenderer.from_config( + mock_renderer = MistralRenderer( MockVllmConfig(mock_model_config), - tokenizer_kwargs={}, + tokenizer=mock_tokenizer, ) - mock_renderer._tokenizer = mock_tokenizer task = mock_renderer.render_messages_async([], ChatParams()) From 8ae37b3e53e84fd25c68fc7169581f760bc09a64 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 14 Feb 2026 05:06:13 +0000 Subject: [PATCH 15/23] Fix Signed-off-by: DarkLight1337 --- vllm/renderers/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 2af0fc8756c8..73e076148990 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -67,7 +67,7 @@ def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None: from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry mm_processor_cache = mm_registry.processor_cache_from_config(config) - self._mm_processor = mm_registry.create_processor( + self.mm_processor = mm_registry.create_processor( config.model_config, config.observability_config, tokenizer=self.tokenizer, @@ -92,10 +92,10 @@ def get_async_tokenizer(self) -> AsyncMicrobatchTokenizer: return self._async_tokenizer def get_mm_processor(self) -> "BaseMultiModalProcessor": - if self._mm_processor is None: + if self.mm_processor is None: raise ValueError("Multi-modal processor not available for text-only models") - return self._mm_processor + return self.mm_processor def stat_mm_cache(self) -> MultiModalCacheStats | None: mm_cache_stats = self._mm_cache_stats From 2b8a46c2e329106ab086d457ef28dd6f5ec4e1a1 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 14 Feb 2026 09:21:56 +0000 Subject: [PATCH 16/23] Fix timeout Signed-off-by: DarkLight1337 --- vllm/renderers/base.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 73e076148990..bd60450ff805 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -12,6 +12,7 @@ from vllm.logger import init_logger from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import AsyncMicrobatchTokenizer +from vllm.utils.torch_utils import set_default_torch_num_threads from vllm.v1.metrics.stats import MultiModalCacheStats from .embed_utils import safe_load_prompt_embeds @@ -61,22 +62,22 @@ def __init__(self, config: "VllmConfig", tokenizer: _T | None) -> None: self._async_tokenizer: AsyncMicrobatchTokenizer | None = None self.mm_processor: BaseMultiModalProcessor | None = None - self._mm_processor_cache: BaseMultiModalProcessorCache | None = None self._mm_cache_stats: MultiModalCacheStats | None = None if config.model_config.is_multimodal_model: from vllm.multimodal import MULTIMODAL_REGISTRY as mm_registry mm_processor_cache = mm_registry.processor_cache_from_config(config) - self.mm_processor = mm_registry.create_processor( - config.model_config, - config.observability_config, - tokenizer=self.tokenizer, - cache=mm_processor_cache, - ) - self._mm_processor_cache = mm_processor_cache - self._mm_cache_stats = ( - MultiModalCacheStats() if mm_processor_cache else None - ) + + with set_default_torch_num_threads(): + self.mm_processor = mm_registry.create_processor( + config.model_config, + config.observability_config, + tokenizer=tokenizer, + cache=mm_processor_cache, + ) + + if mm_processor_cache: + self._mm_cache_stats = MultiModalCacheStats() def get_tokenizer(self) -> _T: tokenizer = self.tokenizer @@ -97,6 +98,13 @@ def get_mm_processor(self) -> "BaseMultiModalProcessor": return self.mm_processor + @property + def mm_processor_cache(self) -> "BaseMultiModalProcessorCache | None": + if self.mm_processor is None: + return None + + return self.mm_processor.cache + def stat_mm_cache(self) -> MultiModalCacheStats | None: mm_cache_stats = self._mm_cache_stats if mm_cache_stats is None: @@ -107,7 +115,7 @@ def stat_mm_cache(self) -> MultiModalCacheStats | None: return mm_cache_stats def update_mm_cache_stats(self) -> None: - mm_processor_cache = self._mm_processor_cache + mm_processor_cache = self.mm_processor_cache mm_cache_stats = self._mm_cache_stats if mm_processor_cache and mm_cache_stats: @@ -115,7 +123,7 @@ def update_mm_cache_stats(self) -> None: mm_cache_stats.record(delta.total, delta.hits) def clear_mm_cache(self) -> None: - mm_processor_cache = self._mm_processor_cache + mm_processor_cache = self.mm_processor_cache if mm_processor_cache is not None: mm_processor_cache.clear_cache() @@ -123,7 +131,7 @@ def clear_mm_cache(self) -> None: self._mm_cache_stats.reset = True def shutdown(self) -> None: - mm_processor_cache = self._mm_processor_cache + mm_processor_cache = self.mm_processor_cache if mm_processor_cache is not None: mm_processor_cache.close() From c1618ac618c0b0482ff8436e6034953e55b1780f Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 14 Feb 2026 18:59:12 +0800 Subject: [PATCH 17/23] Update vllm/model_executor/models/siglip.py Signed-off-by: Cyrus Leung --- vllm/model_executor/models/siglip.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 68aa7f34b793..41334abf6ea4 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -112,9 +112,6 @@ def get_vision_encoder_info(self): def get_hf_processor(self, **kwargs: object): return self.ctx.get_hf_processor(SiglipProcessor, **kwargs) - def get_default_tok_params(self) -> TokenizeParams: - return super().get_default_tok_params().with_kwargs(add_special_tokens=False) - def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": 1} From 1472641afb25a84c1bc65f5c5dccbdf8e3cc6e53 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 14 Feb 2026 18:59:59 +0800 Subject: [PATCH 18/23] Update vllm/model_executor/models/clip.py Signed-off-by: Cyrus Leung --- vllm/model_executor/models/clip.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 3b654b0fd43a..251ed8f0a2b3 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -127,9 +127,6 @@ def get_vision_encoder_info(self): def get_hf_processor(self, **kwargs: object): return self.ctx.get_hf_processor(CLIPProcessor, **kwargs) - def get_default_tok_params(self) -> TokenizeParams: - return super().get_default_tok_params().with_kwargs(add_special_tokens=False) - def get_supported_mm_limits(self) -> Mapping[str, int | None]: return {"image": 1} From 79173c4b37bef45b93b0fba2981808359ac22cef Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 14 Feb 2026 19:01:13 +0800 Subject: [PATCH 19/23] Update vllm/model_executor/models/clip.py Signed-off-by: Cyrus Leung --- vllm/model_executor/models/clip.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 251ed8f0a2b3..3f189eacce7a 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -47,7 +47,6 @@ PromptReplacement, PromptUpdate, ) -from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape From 96b0d2fc4227d238edd51c769bf80b3b278ff153 Mon Sep 17 00:00:00 2001 From: Cyrus Leung Date: Sat, 14 Feb 2026 19:01:20 +0800 Subject: [PATCH 20/23] Update vllm/model_executor/models/siglip.py Signed-off-by: Cyrus Leung --- vllm/model_executor/models/siglip.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 41334abf6ea4..92ecc75793ce 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -53,7 +53,6 @@ PromptReplacement, PromptUpdate, ) -from vllm.renderers import TokenizeParams from vllm.sequence import IntermediateTensors from vllm.utils.tensor_schema import TensorSchema, TensorShape From f107b21df27d1f2b6ed4f6985f12408cf80908f7 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 14 Feb 2026 14:21:54 +0000 Subject: [PATCH 21/23] Less strict Signed-off-by: DarkLight1337 --- vllm/model_executor/models/clip.py | 22 +++++++++++++++------- vllm/model_executor/models/siglip.py | 22 +++++++++++++++------- 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 3f189eacce7a..17f9bbc0d1cf 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -208,14 +208,22 @@ def apply( *, mm_uuids: MultiModalUUIDDict | None = None, ) -> MultiModalInputs: - if prompt and mm_items: - raise ValueError( - "CLIP accepts text-only or image-only inputs, not both! " - "Image-only inputs means passing an image with an empty text " - "prompt." - ) - if mm_items: + if isinstance(prompt, str): + raise ValueError( + "CLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty text prompt." + ) + + special_tokens = self.info.get_tokenizer().all_special_ids + if all(tok in special_tokens for tok in prompt): + prompt = [] + else: + raise ValueError( + "CLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty token prompt." + ) + # For multi-modal data, the prompt after processing should # only contain the dummy image tokens tokenization_kwargs = { diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 92ecc75793ce..8d2a3034c5b2 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -194,14 +194,22 @@ def apply( *, mm_uuids: MultiModalUUIDDict | None = None, ) -> MultiModalInputs: - if prompt and mm_items: - raise ValueError( - "Siglip accepts text-only or image-only inputs, not both! " - "Image-only inputs means passing an image with an empty text " - "prompt." - ) - if mm_items: + if isinstance(prompt, str): + raise ValueError( + "SigLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty text prompt." + ) + + special_tokens = self.info.get_tokenizer().all_special_ids + if all(tok in special_tokens for tok in prompt): + prompt = [] + else: + raise ValueError( + "SigLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty token prompt." + ) + # For multi-modal data, the prompt after processing should # only contain the image token tokenization_kwargs = { From dbd83b19b593a9c8c7438af81804ec209715a157 Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 14 Feb 2026 15:41:42 +0000 Subject: [PATCH 22/23] Fix Signed-off-by: DarkLight1337 --- vllm/model_executor/models/clip.py | 2 +- vllm/model_executor/models/siglip.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index 17f9bbc0d1cf..a27091fa9d41 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -209,7 +209,7 @@ def apply( mm_uuids: MultiModalUUIDDict | None = None, ) -> MultiModalInputs: if mm_items: - if isinstance(prompt, str): + if isinstance(prompt, str) and len(prompt) > 0: raise ValueError( "CLIP accepts text-only or image-only inputs, not both! " "You must pass an image with an empty text prompt." diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index 8d2a3034c5b2..e8c02cd4d951 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -195,7 +195,7 @@ def apply( mm_uuids: MultiModalUUIDDict | None = None, ) -> MultiModalInputs: if mm_items: - if isinstance(prompt, str): + if isinstance(prompt, str) and len(prompt) > 0: raise ValueError( "SigLIP accepts text-only or image-only inputs, not both! " "You must pass an image with an empty text prompt." From 4ba0b3c3ceff21b6a71bb2cb4a3b35703ffd342a Mon Sep 17 00:00:00 2001 From: DarkLight1337 Date: Sat, 14 Feb 2026 15:56:25 +0000 Subject: [PATCH 23/23] Fix Signed-off-by: DarkLight1337 --- vllm/model_executor/models/clip.py | 27 ++++++++++++++------------- vllm/model_executor/models/siglip.py | 27 ++++++++++++++------------- 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/vllm/model_executor/models/clip.py b/vllm/model_executor/models/clip.py index a27091fa9d41..37888086b683 100644 --- a/vllm/model_executor/models/clip.py +++ b/vllm/model_executor/models/clip.py @@ -209,20 +209,21 @@ def apply( mm_uuids: MultiModalUUIDDict | None = None, ) -> MultiModalInputs: if mm_items: - if isinstance(prompt, str) and len(prompt) > 0: - raise ValueError( - "CLIP accepts text-only or image-only inputs, not both! " - "You must pass an image with an empty text prompt." - ) - - special_tokens = self.info.get_tokenizer().all_special_ids - if all(tok in special_tokens for tok in prompt): - prompt = [] + if isinstance(prompt, str): + if len(prompt) > 0: + raise ValueError( + "CLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty text prompt." + ) else: - raise ValueError( - "CLIP accepts text-only or image-only inputs, not both! " - "You must pass an image with an empty token prompt." - ) + special_tokens = self.info.get_tokenizer().all_special_ids + if all(tok in special_tokens for tok in prompt): + prompt = [] + else: + raise ValueError( + "CLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty token prompt." + ) # For multi-modal data, the prompt after processing should # only contain the dummy image tokens diff --git a/vllm/model_executor/models/siglip.py b/vllm/model_executor/models/siglip.py index e8c02cd4d951..a447d376b220 100644 --- a/vllm/model_executor/models/siglip.py +++ b/vllm/model_executor/models/siglip.py @@ -195,20 +195,21 @@ def apply( mm_uuids: MultiModalUUIDDict | None = None, ) -> MultiModalInputs: if mm_items: - if isinstance(prompt, str) and len(prompt) > 0: - raise ValueError( - "SigLIP accepts text-only or image-only inputs, not both! " - "You must pass an image with an empty text prompt." - ) - - special_tokens = self.info.get_tokenizer().all_special_ids - if all(tok in special_tokens for tok in prompt): - prompt = [] + if isinstance(prompt, str): + if len(prompt) > 0: + raise ValueError( + "SigLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty text prompt." + ) else: - raise ValueError( - "SigLIP accepts text-only or image-only inputs, not both! " - "You must pass an image with an empty token prompt." - ) + special_tokens = self.info.get_tokenizer().all_special_ids + if all(tok in special_tokens for tok in prompt): + prompt = [] + else: + raise ValueError( + "SigLIP accepts text-only or image-only inputs, not both! " + "You must pass an image with an empty token prompt." + ) # For multi-modal data, the prompt after processing should # only contain the image token