From 9b8363fe568fa3b290bb6ea724d741c880b87947 Mon Sep 17 00:00:00 2001 From: Jan Kwiecinski Date: Sat, 5 Sep 2026 16:38:29 -0700 Subject: [PATCH 1/9] Test that video loading is not skipped in the current implementation Signed-off-by: Jan Kwiecinski --- .../test_process_multi_modal_uuids.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index 76b394897ee1..41b296364079 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from http import HTTPStatus +from unittest.mock import Mock import pytest @@ -9,8 +10,10 @@ from vllm.assets.video import VideoAsset from vllm.config import CacheConfig, ModelConfig, VllmConfig from vllm.entrypoints.serve import create_error_response +from vllm.multimodal.media import MediaConnector from vllm.multimodal.parse import parse_mm_uuids from vllm.renderers.hf import HfRenderer +from vllm.renderers.params import ChatParams from vllm.tokenizers.registry import cached_tokenizer_from_config cherry_pil_image = ImageAsset("cherry_blossom").pil_image @@ -64,6 +67,47 @@ def test_text_only_model_mm_data_maps_to_bad_request(): assert error_response.error.code == HTTPStatus.BAD_REQUEST +@pytest.mark.parametrize( + ("modality", "media"), + [ + pytest.param("image", cherry_pil_image, id="image"), + pytest.param("video", baby_reading_np_ndarrays, id="video"), + ], +) +def test_cached_uuid_skips_url_loading( + monkeypatch: pytest.MonkeyPatch, + modality: str, + media: object, +): + renderer = _build_renderer() + media_url = f"https://example.com/test.{modality}" + media_uuid = f"test-{modality}-uuid" + fetch_media = Mock(return_value=media) + monkeypatch.setattr(MediaConnector, f"fetch_{modality}", fetch_media) + + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"Describe this {modality}."}, + { + "type": f"{modality}_url", + f"{modality}_url": {"url": media_url}, + "uuid": media_uuid, + }, + ], + } + ] + + _, first_prompts = renderer.render_chat([messages], ChatParams()) + _, second_prompts = renderer.render_chat([messages], ChatParams()) + + first_input = first_prompts[0] + second_input = second_prompts[0] + assert first_input["mm_hashes"] == second_input["mm_hashes"] + assert fetch_media.call_count == 1 + + def test_multi_modal_uuids_length_mismatch_raises(): renderer = _build_renderer() From 1a6a430ac02e621c1d1cd958bd236d4e6be748f0 Mon Sep 17 00:00:00 2001 From: Jan Kwiecinski Date: Sun, 6 Sep 2026 06:33:22 -0700 Subject: [PATCH 2/9] Make it possible to skip media loadingfor video and images Signed-off-by: Jan Kwiecinski --- .../test_process_multi_modal_uuids.py | 1 + vllm/entrypoints/chat_utils.py | 72 ++++++++++- vllm/envs.py | 7 ++ vllm/multimodal/processing/inputs.py | 115 ++++++++++-------- vllm/renderers/base.py | 31 ++++- vllm/renderers/hf.py | 36 ++++++ 6 files changed, 203 insertions(+), 59 deletions(-) diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index 41b296364079..2979d4c84ae5 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -79,6 +79,7 @@ def test_cached_uuid_skips_url_loading( modality: str, media: object, ): + monkeypatch.setenv("VLLM_EARLY_UUID_LOOKUPS", "1") renderer = _build_renderer() media_url = f"https://example.com/test.{modality}" media_uuid = f"test-{modality}-uuid" diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index fe0eac85b1f8..4281cf4cad15 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -6,7 +6,7 @@ import types from abc import ABC, abstractmethod from collections import Counter, defaultdict -from collections.abc import Awaitable, Callable, Iterable +from collections.abc import Awaitable, Callable, Iterable, Mapping from dataclasses import dataclass from functools import cached_property, lru_cache, partial from itertools import accumulate @@ -57,6 +57,7 @@ from vllm.logger import init_logger from vllm.model_executor.models import SupportsMultiModal from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.cache import BaseMultiModalProcessorCache from vllm.multimodal.inputs import ( MultiModalBatchedField, MultiModalFlatField, @@ -66,7 +67,7 @@ VisionChunkVideo, ) from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector -from vllm.multimodal.processing import BaseMultiModalProcessor +from vllm.multimodal.processing import BaseMultiModalProcessor, ProcessorInputs from vllm.renderers.embed_utils import ( safe_load_prompt_embeds, safe_load_prompt_embeds_async, @@ -560,11 +561,15 @@ def __init__( self, model_config: ModelConfig, media_io_kwargs: dict[str, dict[str, Any]] | None = None, + mm_processor_cache: BaseMultiModalProcessorCache | None = None, + skip_mm_cache: bool = False, ): super().__init__() self._model_config = model_config self._media_io_kwargs = media_io_kwargs + self._mm_processor_cache = mm_processor_cache + self._skip_mm_cache = skip_mm_cache self._items_by_modality = defaultdict[str, list[_T]](list) # Track original modality for each vision_chunk item (image or video) @@ -610,6 +615,33 @@ def mm_registry(self): def mm_processor(self): return self.mm_registry.create_processor(self.model_config) + def is_mm_item_cached( + self, + modality: str, + uuid: str | None, + mm_processor_kwargs: Mapping[str, object] | None, + ) -> bool: + cache = self._mm_processor_cache + if not ( + envs.VLLM_EARLY_UUID_LOOKUPS + and not self._skip_mm_cache + and uuid is not None + and cache is not None + ): + return False + + mm_config = self.model_config.get_multimodal_config() + mm_hash = ProcessorInputs.get_mm_item_hash( + modality, + None, + uuid, + self.model_config.model, + mm_config.mm_hasher_algorithm, + mm_processor_kwargs or {}, + self.media_io_kwargs or {}, + ) + return cache.is_cached_item(mm_hash) + @property def video_processor_name(self) -> str | None: return get_video_processor_cls_name(self.model_config) @@ -1036,6 +1068,10 @@ def parse_prompt_embeds(self, data: str) -> None: self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN) def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: + if self._tracker.is_mm_item_cached( + "image", uuid, self._mm_processor_kwargs + ): + image_url = None image = self._connector.fetch_image(image_url) if image_url else None placeholder = self._tracker.add("image", (image, uuid)) @@ -1124,12 +1160,19 @@ def parse_input_audio( return self.parse_audio(audio_url, uuid) def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: + video_url_to_fetch = ( + None + if self._tracker.is_mm_item_cached( + "video", uuid, self._mm_processor_kwargs + ) + else video_url + ) video = ( self._connector.fetch_video( - video_url=video_url, + video_url=video_url_to_fetch, video_processor=self._tracker.video_processor_name, ) - if video_url + if video_url_to_fetch else None ) @@ -1235,6 +1278,10 @@ async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None): return image, uuid def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: + if self._tracker.is_mm_item_cached( + "image", uuid, self._mm_processor_kwargs + ): + image_url = None placeholder = self._tracker.add( "image", partial(self._image_with_uuid_async, image_url, uuid) ) @@ -1364,8 +1411,15 @@ async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None): return video, uuid def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: + video_url_to_fetch = ( + None + if self._tracker.is_mm_item_cached( + "video", uuid, self._mm_processor_kwargs + ) + else video_url + ) placeholder = self._tracker.add( - "video", partial(self._video_with_uuid_async, video_url, uuid) + "video", partial(self._video_with_uuid_async, video_url_to_fetch, uuid) ) self._add_placeholder("video", placeholder) @@ -2123,6 +2177,8 @@ def parse_chat_messages( content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, + mm_processor_cache: BaseMultiModalProcessorCache | None = None, + skip_mm_cache: bool = False, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -2132,6 +2188,8 @@ def parse_chat_messages( mm_tracker = MultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, + mm_processor_cache=mm_processor_cache, + skip_mm_cache=skip_mm_cache, ) for msg in messages: @@ -2162,6 +2220,8 @@ async def parse_chat_messages_async( content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, + mm_processor_cache: BaseMultiModalProcessorCache | None = None, + skip_mm_cache: bool = False, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -2171,6 +2231,8 @@ async def parse_chat_messages_async( mm_tracker = AsyncMultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, + mm_processor_cache=mm_processor_cache, + skip_mm_cache=skip_mm_cache, ) for msg in messages: diff --git a/vllm/envs.py b/vllm/envs.py index a3ef86d1b6b4..ed8028abb417 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -74,6 +74,7 @@ VLLM_MEDIA_CACHE: str = "" VLLM_MEDIA_CACHE_MAX_SIZE_MB: int = 5120 VLLM_MEDIA_CACHE_TTL_HOURS: float = 24 + VLLM_EARLY_UUID_LOOKUPS: bool = False VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3 VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256 VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True @@ -978,6 +979,11 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_MEDIA_CACHE_TTL_HOURS": lambda: float( os.getenv("VLLM_MEDIA_CACHE_TTL_HOURS", "24") ), + # Whether to probe the multi-modal processor cache before resolving media + # sources with UUIDs. Cache hits skip source loading. + "VLLM_EARLY_UUID_LOOKUPS": lambda: bool( + int(os.getenv("VLLM_EARLY_UUID_LOOKUPS", "0")) + ), # Maximum number of retries for fetching media (images, audio, video) # from URLs. Each retry quadruples the timeout. Default is 3. "VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int( @@ -2346,6 +2352,7 @@ def compile_factors() -> dict[str, object]: "VLLM_MEDIA_CACHE", "VLLM_MEDIA_CACHE_MAX_SIZE_MB", "VLLM_MEDIA_CACHE_TTL_HOURS", + "VLLM_EARLY_UUID_LOOKUPS", "VLLM_MEDIA_FETCH_MAX_RETRIES", "VLLM_MEDIA_URL_ALLOW_REDIRECTS", "VLLM_MEDIA_LOADING_THREAD_COUNT", diff --git a/vllm/multimodal/processing/inputs.py b/vllm/multimodal/processing/inputs.py index 56750c3087cb..e277cbcc724b 100644 --- a/vllm/multimodal/processing/inputs.py +++ b/vllm/multimodal/processing/inputs.py @@ -29,6 +29,59 @@ class ProcessorInputs: hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict) media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict) + @staticmethod + def get_mm_item_hash( + modality: str, + item: object, + uuid_item: str | None, + model_id: str, + hash_algorithm: MMHasherAlgorithm, + hf_processor_mm_kwargs: Mapping[str, object], + media_io_kwargs: Mapping[str, Mapping[str, object]], + ) -> str: + # Each cache entry represents one item from one modality. Keep + # modality-scoped options out of unrelated entries, while retaining + # flat processor kwargs because they are shared by HF. + if modality in _HF_MODALITY_PROCESSOR_KWARGS: + modality_media_io_kwargs = media_io_kwargs.get(modality, {}) + scoped_media_io_kwargs = ( + {modality: modality_media_io_kwargs} + if modality_media_io_kwargs + else {} + ) + scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality] + mm_processor_kwargs = { + key: value + for key, value in hf_processor_mm_kwargs.items() + if key not in _HF_MODALITY_PROCESSOR_KWARGS.values() + or key == scoped_key + } + else: + # Unified modalities such as ``vision_chunk`` can contain items + # originating from multiple media modalities. There is no per-item + # origin metadata here, so preserve all factors. + scoped_media_io_kwargs = dict(media_io_kwargs) + mm_processor_kwargs = dict(hf_processor_mm_kwargs) + + hash_factors = { + "media_io_kwargs": scoped_media_io_kwargs, + "mm_processor_kwargs": mm_processor_kwargs, + } + hash_factors = {key: value for key, value in hash_factors.items() if value} + + # Even if a UUID is provided, model output depends on the current + # modality's hash factors, so they are taken into account. + if uuid_item is None or hash_factors: + item = uuid_item if uuid_item is not None else item + return MultiModalHasher.hash_kwargs( + hash_algorithm, + model_id=model_id, + **{modality: item}, + **hash_factors, + ) + + return uuid_item + def get_mm_hashes( self, model_id: str, @@ -38,40 +91,7 @@ def get_mm_hashes( mm_uuid_items = self.mm_uuid_items or {} mm_hashes = dict[str, list[str]]() - hasher = MultiModalHasher - for modality, data_items in mm_data_items.items(): - # Each cache entry represents one item from one modality. Keep - # modality-scoped options out of unrelated entries, while - # retaining flat processor kwargs because they are shared by HF. - if modality in _HF_MODALITY_PROCESSOR_KWARGS: - modality_media_io_kwargs = self.media_io_kwargs.get(modality, {}) - media_io_kwargs = ( - {modality: modality_media_io_kwargs} - if modality_media_io_kwargs - else {} - ) - scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality] - mm_processor_kwargs = { - key: value - for key, value in self.hf_processor_mm_kwargs.items() - if key not in _HF_MODALITY_PROCESSOR_KWARGS.values() - or key == scoped_key - } - else: - # Unified modalities such as ``vision_chunk`` can contain - # items originating from multiple media modalities. There is - # no per-item origin metadata here, so preserve all factors. - media_io_kwargs = dict(self.media_io_kwargs) - mm_processor_kwargs = dict(self.hf_processor_mm_kwargs) - - hash_factors = { - "media_io_kwargs": media_io_kwargs, - "mm_processor_kwargs": mm_processor_kwargs, - } - hash_factors = {key: value for key, value in hash_factors.items() if value} - has_hash_factors = bool(hash_factors) - uuid_items = ( mm_uuid_items[modality] if modality in mm_uuid_items @@ -81,26 +101,17 @@ def get_mm_hashes( # For None entries, compute a hash; otherwise, use provided ID. hashes: list[str] = [] for i, item in enumerate(data_items.get_all_items_for_hash()): - uuid_item = uuid_items[i] - - # NOTE: Even if a uuid_item is provided, model output depends - # on the current modality's hash factors, so they are taken - # into account. - if uuid_item is None or has_hash_factors: - # NOTE: use provided hash string to hash with kwargs - # if available for better performance. - item = uuid_item if uuid_item is not None else item - hashes.append( - hasher.hash_kwargs( - hash_algorithm, - model_id=model_id, - **{modality: item}, - **hash_factors, - ) + hashes.append( + self.get_mm_item_hash( + modality, + item, + uuid_items[i], + model_id, + hash_algorithm, + self.hf_processor_mm_kwargs, + self.media_io_kwargs, ) - else: - # If there are no extra kwargs, use the client-provided UUID. - hashes.append(uuid_item) + ) mm_hashes[modality] = hashes diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 264a811a9cb3..c6685a8ad33b 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -519,6 +519,25 @@ async def render_messages_async( ) -> tuple[list["ConversationMessage"], DictPrompt]: return self.render_messages(messages, params) + # Helpers to enable passing skip_mm_cache to the renderer in some subclasses (e.g. HfRenderer). + def _render_messages_for_chat( + self, + messages: list["ChatCompletionMessageParam"], + params: ChatParams, + *, + skip_mm_cache: bool, + ) -> tuple[list["ConversationMessage"], DictPrompt]: + return self.render_messages(messages, params) + + async def _render_messages_for_chat_async( + self, + messages: list["ChatCompletionMessageParam"], + params: ChatParams, + *, + skip_mm_cache: bool, + ) -> tuple[list["ConversationMessage"], DictPrompt]: + return await self.render_messages_async(messages, params) + # Step 2: Tokenize prompts if necessary def _can_produce_offsets(self) -> bool: """Whether this renderer's tokenizer can emit char-level offsets. @@ -1170,7 +1189,11 @@ def render_chat( tok_params = self.default_chat_tok_params rendered = [ - self.render_messages(conversation, chat_params) + self._render_messages_for_chat( + conversation, + chat_params, + skip_mm_cache=skip_mm_cache, + ) for conversation in conversations ] @@ -1208,7 +1231,11 @@ async def render_chat_async( tok_params = self.default_chat_tok_params rendered = [ - self.render_messages_async(conversation, chat_params) + self._render_messages_for_chat_async( + conversation, + chat_params, + skip_mm_cache=skip_mm_cache, + ) for conversation in conversations ] diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 16906bb1da4f..4b80e66de12c 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -949,10 +949,26 @@ def _can_produce_offsets(self) -> bool: # expose offset_mapping. return self.tokenizer is not None and self.tokenizer.is_fast + @override + def _render_messages_for_chat( + self, + messages: list[ChatCompletionMessageParam], + params: ChatParams, + *, + skip_mm_cache: bool, + ) -> tuple[list[ConversationMessage], DictPrompt]: + return self.render_messages( + messages, + params, + skip_mm_cache=skip_mm_cache, + ) + def render_messages( self, messages: list[ChatCompletionMessageParam], params: ChatParams, + *, + skip_mm_cache: bool = False, ) -> tuple[list[ConversationMessage], DictPrompt]: model_config = self.model_config tokenizer = self.get_tokenizer() @@ -975,6 +991,8 @@ def render_messages( ), media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, + mm_processor_cache=self.mm_processor_cache, + skip_mm_cache=skip_mm_cache, ) # prompt_embeds tensors are carried by the tracker through mm_data, @@ -1069,10 +1087,26 @@ def render_messages( return conversation, prompt + @override + async def _render_messages_for_chat_async( + self, + messages: list[ChatCompletionMessageParam], + params: ChatParams, + *, + skip_mm_cache: bool, + ) -> tuple[list[ConversationMessage], DictPrompt]: + return await self.render_messages_async( + messages, + params, + skip_mm_cache=skip_mm_cache, + ) + async def render_messages_async( self, messages: list[ChatCompletionMessageParam], params: ChatParams, + *, + skip_mm_cache: bool = False, ) -> tuple[list[ConversationMessage], DictPrompt]: model_config = self.model_config tokenizer = self.get_tokenizer() @@ -1095,6 +1129,8 @@ async def render_messages_async( ), media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, + mm_processor_cache=self.mm_processor_cache, + skip_mm_cache=skip_mm_cache, ) prompt_embeds_tensors: list[torch.Tensor] | None = None From 5853f062c80abb339d29b4cd334d8b60e038c3e0 Mon Sep 17 00:00:00 2001 From: Jan Kwiecinski Date: Sun, 6 Sep 2026 07:16:19 -0700 Subject: [PATCH 3/9] Add skip_early_mm_lookup param to opt-out of early mm lookups per request Signed-off-by: Jan Kwiecinski --- .../openai/responses/test_protocol.py | 10 ++++++++++ .../test_process_multi_modal_uuids.py | 20 +++++++++++++------ vllm/entrypoints/chat_utils.py | 14 ++++++------- .../openai/chat_completion/protocol.py | 6 ++++++ vllm/entrypoints/openai/responses/protocol.py | 5 +++++ vllm/renderers/hf.py | 4 ++-- vllm/renderers/params.py | 4 ++++ 7 files changed, 48 insertions(+), 15 deletions(-) diff --git a/tests/entrypoints/openai/responses/test_protocol.py b/tests/entrypoints/openai/responses/test_protocol.py index db5d7d692490..fad8e6863ed0 100644 --- a/tests/entrypoints/openai/responses/test_protocol.py +++ b/tests/entrypoints/openai/responses/test_protocol.py @@ -5,6 +5,7 @@ ) from vllm.entrypoints.openai.responses.protocol import ( + ResponsesRequest, serialize_message, serialize_messages, ) @@ -37,3 +38,12 @@ def test_serialize_messages() -> None: } msg = Message.from_dict(msg_value) assert serialize_messages([msg, dict_value]) == [msg_value, dict_value] + + +def test_skip_early_mm_lookup_is_preserved_in_chat_params() -> None: + request = ResponsesRequest(input="hello", skip_early_mm_lookup=True) + + chat_params = request.build_chat_params(None, "auto") + + assert request.skip_early_mm_lookup is True + assert chat_params.skip_early_mm_lookup is True diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index 2979d4c84ae5..703de9986b35 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -68,16 +68,23 @@ def test_text_only_model_mm_data_maps_to_bad_request(): @pytest.mark.parametrize( - ("modality", "media"), + ("modality", "media", "skip_early_mm_lookup"), [ - pytest.param("image", cherry_pil_image, id="image"), - pytest.param("video", baby_reading_np_ndarrays, id="video"), + pytest.param("image", cherry_pil_image, False, id="image"), + pytest.param("video", baby_reading_np_ndarrays, False, id="video"), + pytest.param( + "video", + baby_reading_np_ndarrays, + True, + id="video-skip-early-mm-lookup", + ), ], ) def test_cached_uuid_skips_url_loading( monkeypatch: pytest.MonkeyPatch, modality: str, media: object, + skip_early_mm_lookup: bool, ): monkeypatch.setenv("VLLM_EARLY_UUID_LOOKUPS", "1") renderer = _build_renderer() @@ -100,13 +107,14 @@ def test_cached_uuid_skips_url_loading( } ] - _, first_prompts = renderer.render_chat([messages], ChatParams()) - _, second_prompts = renderer.render_chat([messages], ChatParams()) + params = ChatParams(skip_early_mm_lookup=skip_early_mm_lookup) + _, first_prompts = renderer.render_chat([messages], params) + _, second_prompts = renderer.render_chat([messages], params) first_input = first_prompts[0] second_input = second_prompts[0] assert first_input["mm_hashes"] == second_input["mm_hashes"] - assert fetch_media.call_count == 1 + assert fetch_media.call_count == (2 if skip_early_mm_lookup else 1) def test_multi_modal_uuids_length_mismatch_raises(): diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 4281cf4cad15..c62c1942ce4b 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -562,14 +562,14 @@ def __init__( model_config: ModelConfig, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_cache: BaseMultiModalProcessorCache | None = None, - skip_mm_cache: bool = False, + skip_early_mm_lookup: bool = False, ): super().__init__() self._model_config = model_config self._media_io_kwargs = media_io_kwargs self._mm_processor_cache = mm_processor_cache - self._skip_mm_cache = skip_mm_cache + self._skip_early_mm_lookup = skip_early_mm_lookup self._items_by_modality = defaultdict[str, list[_T]](list) # Track original modality for each vision_chunk item (image or video) @@ -624,7 +624,7 @@ def is_mm_item_cached( cache = self._mm_processor_cache if not ( envs.VLLM_EARLY_UUID_LOOKUPS - and not self._skip_mm_cache + and not self._skip_early_mm_lookup and uuid is not None and cache is not None ): @@ -2178,7 +2178,7 @@ def parse_chat_messages( media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, mm_processor_cache: BaseMultiModalProcessorCache | None = None, - skip_mm_cache: bool = False, + skip_early_mm_lookup: bool = False, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -2189,7 +2189,7 @@ def parse_chat_messages( model_config, media_io_kwargs=media_io_kwargs, mm_processor_cache=mm_processor_cache, - skip_mm_cache=skip_mm_cache, + skip_early_mm_lookup=skip_early_mm_lookup, ) for msg in messages: @@ -2221,7 +2221,7 @@ async def parse_chat_messages_async( media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, mm_processor_cache: BaseMultiModalProcessorCache | None = None, - skip_mm_cache: bool = False, + skip_early_mm_lookup: bool = False, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -2232,7 +2232,7 @@ async def parse_chat_messages_async( model_config, media_io_kwargs=media_io_kwargs, mm_processor_cache=mm_processor_cache, - skip_mm_cache=skip_mm_cache, + skip_early_mm_lookup=skip_early_mm_lookup, ) for msg in messages: diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 1a49206316ef..215b38325a30 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -372,6 +372,10 @@ class ChatCompletionRequest(OpenAIBaseModel): default=None, description=("Additional kwargs to pass to the HF processor."), ) + skip_early_mm_lookup: bool = Field( + default=False, + description="Skip early UUID-based multi-modal cache lookups.", + ) structured_outputs: StructuredOutputsParams | None = Field( default=None, description="Additional kwargs for structured outputs", @@ -595,6 +599,7 @@ def build_chat_params( extra_kwargs, ), media_io_kwargs=self.media_io_kwargs, + skip_early_mm_lookup=self.skip_early_mm_lookup, return_assistant_tokens_mask=bool(self.return_assistant_tokens_mask), # No-tools requests default to tool_choice="none" at the API # layer. Collapse that default before rendering, so K3 emits a @@ -1107,6 +1112,7 @@ class BatchChatCompletionRequest(OpenAIBaseModel): chat_template_kwargs: dict[str, Any] | None = None media_io_kwargs: dict[str, dict[str, Any]] | None = None mm_processor_kwargs: dict[str, Any] | None = None + skip_early_mm_lookup: bool = False priority: int = Field(default=0, ge=_INT64_MIN, le=_INT64_MAX) cache_salt: str | None = None include_stop_str_in_output: bool = False diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 0363c01de310..33d2362ffc7b 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -240,6 +240,10 @@ class ResponsesRequest(OpenAIBaseModel): default=None, description=("Additional kwargs to pass to the HF processor."), ) + skip_early_mm_lookup: bool = Field( + default=False, + description="Skip early UUID-based multi-modal cache lookups.", + ) priority: int = Field( default=0, ge=_INT64_MIN, @@ -348,6 +352,7 @@ def build_chat_params( extra_kwargs, ), media_io_kwargs=self.media_io_kwargs, + skip_early_mm_lookup=self.skip_early_mm_lookup, tool_choice=self.tool_choice if self.tools else None, ) diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 4b80e66de12c..8cbbfe3731e7 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -992,7 +992,7 @@ def render_messages( media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, mm_processor_cache=self.mm_processor_cache, - skip_mm_cache=skip_mm_cache, + skip_early_mm_lookup=(params.skip_early_mm_lookup or skip_mm_cache), ) # prompt_embeds tensors are carried by the tracker through mm_data, @@ -1130,7 +1130,7 @@ async def render_messages_async( media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, mm_processor_cache=self.mm_processor_cache, - skip_mm_cache=skip_mm_cache, + skip_early_mm_lookup=(params.skip_early_mm_lookup or skip_mm_cache), ) prompt_embeds_tensors: list[torch.Tensor] | None = None diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index e0eb023dff43..15271a1add04 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -99,6 +99,9 @@ class ChatParams: mm_processor_kwargs: dict[str, Any] | None = None """The kwargs to pass to the multi-modal processor.""" + skip_early_mm_lookup: bool = False + """Skip early UUID-based multi-modal cache lookups.""" + return_assistant_tokens_mask: bool = False """Request a per-token assistant mask from apply_chat_template.""" @@ -136,6 +139,7 @@ def with_defaults( default_mm_processor_kwargs, self.mm_processor_kwargs, ), + skip_early_mm_lookup=self.skip_early_mm_lookup, return_assistant_tokens_mask=self.return_assistant_tokens_mask, tool_choice=self.tool_choice, response_format=self.response_format, From 0bade851beb02aaf8d1afd9675f1a19b49a41519 Mon Sep 17 00:00:00 2001 From: Jan Kwiecinski Date: Sun, 6 Sep 2026 07:38:14 -0700 Subject: [PATCH 4/9] Add docs for change Signed-off-by: Jan Kwiecinski --- docs/features/multimodal_inputs.md | 16 ++++++++++++++++ .../openai/chat_completion/protocol.py | 5 ++++- vllm/entrypoints/openai/responses/protocol.py | 5 ++++- vllm/envs.py | 5 +++-- 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index a36e40c3b962..dee8f1cc19f5 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1374,3 +1374,19 @@ Just like with offline inference, you can skip sending media if you expect cache }, ``` + +#### Early UUID Cache Lookups + +When using the HF renderer, vLLM can probe the multi-modal processor cache +before loading an image or video URL. Off by default, opt in via `VLLM_EARLY_UUID_LOOKUPS=1`. + +The first request with an image or video URL and UUID loads and processes the +media as usual. Repeated requests probe multimodal cache first: +* ***On a hit*** vLLM skips loading and decoding the URL. +* ***On a cache miss*** vLLM uses the URL as a fallback and processes the media normally. + +A race condition is possible if the multimodal cache is evicted between the early lookup and media loading. +Use `skip_early_mm_lookup` to opt-out per request. + +The UUID is authoritative on a cache hit; vLLM does not verify that the URL +still identifies the cached media (see [Multimodal Media UUID Security](../usage/security.md#multimodal-media-uuid-security)) \ No newline at end of file diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 215b38325a30..088cf368f3a1 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -374,7 +374,10 @@ class ChatCompletionRequest(OpenAIBaseModel): ) skip_early_mm_lookup: bool = Field( default=False, - description="Skip early UUID-based multi-modal cache lookups.", + description=( + "Skip early image and video UUID cache lookups for this request. " + "Only applies when VLLM_EARLY_UUID_LOOKUPS=1." + ), ) structured_outputs: StructuredOutputsParams | None = Field( default=None, diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 33d2362ffc7b..05048c2a8e6d 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -242,7 +242,10 @@ class ResponsesRequest(OpenAIBaseModel): ) skip_early_mm_lookup: bool = Field( default=False, - description="Skip early UUID-based multi-modal cache lookups.", + description=( + "Skip early image and video UUID cache lookups for this request. " + "Only applies when VLLM_EARLY_UUID_LOOKUPS=1." + ), ) priority: int = Field( default=0, diff --git a/vllm/envs.py b/vllm/envs.py index ed8028abb417..5b020164488e 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -979,8 +979,9 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_MEDIA_CACHE_TTL_HOURS": lambda: float( os.getenv("VLLM_MEDIA_CACHE_TTL_HOURS", "24") ), - # Whether to probe the multi-modal processor cache before resolving media - # sources with UUIDs. Cache hits skip source loading. + # Whether to probe the multi-modal processor cache before loading image or + # video URLs with UUIDs. Cache hits skip URL loading and decoding. Disabled + # by default. "VLLM_EARLY_UUID_LOOKUPS": lambda: bool( int(os.getenv("VLLM_EARLY_UUID_LOOKUPS", "0")) ), From 166ffdb5415b69d253ee8aa1baae18589027df4b Mon Sep 17 00:00:00 2001 From: Jan Kwiecinski Date: Sun, 6 Sep 2026 15:08:22 -0700 Subject: [PATCH 5/9] Disable early lookups for vision chunks Signed-off-by: Jan Kwiecinski --- .../test_process_multi_modal_uuids.py | 72 ++++++++++++++++++- vllm/entrypoints/chat_utils.py | 1 + 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index 703de9986b35..03d0d347c3eb 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -5,6 +5,7 @@ from unittest.mock import Mock import pytest +from PIL import Image from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset @@ -19,15 +20,34 @@ cherry_pil_image = ImageAsset("cherry_blossom").pil_image stop_pil_image = ImageAsset("stop_sign").pil_image baby_reading_np_ndarrays = VideoAsset("baby_reading").np_ndarrays +kimi_vision_chunk_image = { + "type": "image", + "image": cherry_pil_image, + "uuid": "test-image-uuid", +} +kimi_vision_chunk_video = { + "type": "video_chunk", + "video_chunk": [Image.fromarray(frame) for frame in baby_reading_np_ndarrays[:4]], + "uuid": "test-video-uuid", + "video_idx": 0, + "prompt": ( + "<|media_begin|>video<|media_content|><|media_pad|><|media_end|>" + ), +} def _build_renderer( - *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True + *, + model: str = "Qwen/Qwen2.5-VL-3B-Instruct", + mm_cache_gb: float = 4.0, + enable_prefix_caching: bool = True, + trust_remote_code: bool = False, ) -> HfRenderer: model_config = ModelConfig( - model="Qwen/Qwen2.5-VL-3B-Instruct", + model=model, max_model_len=128, mm_processor_cache_gb=mm_cache_gb, + trust_remote_code=trust_remote_code, ) vllm_config = VllmConfig( @@ -117,6 +137,54 @@ def test_cached_uuid_skips_url_loading( assert fetch_media.call_count == (2 if skip_early_mm_lookup else 1) +@pytest.mark.parametrize( + ("modality", "media"), + [ + pytest.param("image", kimi_vision_chunk_image, id="image"), + pytest.param("video", kimi_vision_chunk_video, id="video"), + ], +) +def test_early_uuid_lookup_is_disabled_for_unified_vision_chunks( + monkeypatch: pytest.MonkeyPatch, + modality: str, + media: object, +): + monkeypatch.setenv("VLLM_EARLY_UUID_LOOKUPS", "1") + renderer = _build_renderer( + model="moonshotai/Kimi-K2.5", + trust_remote_code=True, + ) + mm_processor_cache = renderer.mm_processor_cache + assert mm_processor_cache is not None + + is_cached_item = Mock(return_value=True) + monkeypatch.setattr(mm_processor_cache, "is_cached_item", is_cached_item) + fetch_media = Mock(return_value=media) + monkeypatch.setattr(MediaConnector, f"fetch_{modality}", fetch_media) + + media_url = f"https://example.com/test.{modality}" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"Describe this {modality}."}, + { + "type": f"{modality}_url", + f"{modality}_url": {"url": media_url}, + "uuid": f"test-{modality}-uuid", + }, + ], + } + ] + + conversation, prompt = renderer.render_messages(messages, ChatParams()) + + assert len(conversation) == 1 + assert prompt["multi_modal_data"] == {"vision_chunk": [media]} + assert fetch_media.call_count == 1 + is_cached_item.assert_not_called() + + def test_multi_modal_uuids_length_mismatch_raises(): renderer = _build_renderer() diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 96ab25969cb3..7896db27d51c 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -625,6 +625,7 @@ def is_mm_item_cached( if not ( envs.VLLM_EARLY_UUID_LOOKUPS and not self._skip_early_mm_lookup + and not self.use_unified_vision_chunk_modality and uuid is not None and cache is not None ): From 3d287759941f71ed95c882aa10c3e2c2373fd355 Mon Sep 17 00:00:00 2001 From: Jan Kwiecinski Date: Fri, 11 Sep 2026 09:00:11 -0700 Subject: [PATCH 6/9] Remove env var and retry race condition failures Signed-off-by: Jan Kwiecinski --- docs/features/multimodal_inputs.md | 8 +- .../test_process_multi_modal_uuids.py | 56 +++++- vllm/entrypoints/chat_utils.py | 125 ++++++++----- .../openai/chat_completion/protocol.py | 5 +- vllm/entrypoints/openai/responses/protocol.py | 5 +- vllm/envs.py | 8 - vllm/multimodal/processing/processor.py | 28 ++- vllm/renderers/base.py | 166 ++++++++++++++++-- vllm/renderers/hf.py | 9 + 9 files changed, 318 insertions(+), 92 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 32fe1d554c5b..adf56cb28699 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1411,16 +1411,16 @@ Just like with offline inference, you can skip sending media if you expect cache #### Early UUID Cache Lookups -When using the HF renderer, vLLM can probe the multi-modal processor cache -before loading an image or video URL. Off by default, opt in via `VLLM_EARLY_UUID_LOOKUPS=1`. +When using the HF renderer, vLLM probes the multi-modal processor cache before +loading an image or video URL with a UUID. Use `skip_early_mm_lookup` to opt out per request. The first request with an image or video URL and UUID loads and processes the media as usual. Repeated requests probe multimodal cache first: * ***On a hit*** vLLM skips loading and decoding the URL. * ***On a cache miss*** vLLM uses the URL as a fallback and processes the media normally. -A race condition is possible if the multimodal cache is evicted between the early lookup and media loading. -Use `skip_early_mm_lookup` to opt-out per request. +If some cached items are evicted between the early lookup and multi-modal processing, +vLLM loads the retained URLs and retries processing only the missing items. The UUID is authoritative on a cache hit; vLLM does not verify that the URL still identifies the cached media (see [Multimodal Media UUID Security](../usage/security.md#multimodal-media-uuid-security)) \ No newline at end of file diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index 03d0d347c3eb..6a1c102c4ea8 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -30,9 +30,7 @@ "video_chunk": [Image.fromarray(frame) for frame in baby_reading_np_ndarrays[:4]], "uuid": "test-video-uuid", "video_idx": 0, - "prompt": ( - "<|media_begin|>video<|media_content|><|media_pad|><|media_end|>" - ), + "prompt": ("<|media_begin|>video<|media_content|><|media_pad|><|media_end|>"), } @@ -106,7 +104,6 @@ def test_cached_uuid_skips_url_loading( media: object, skip_early_mm_lookup: bool, ): - monkeypatch.setenv("VLLM_EARLY_UUID_LOOKUPS", "1") renderer = _build_renderer() media_url = f"https://example.com/test.{modality}" media_uuid = f"test-{modality}-uuid" @@ -137,6 +134,56 @@ def test_cached_uuid_skips_url_loading( assert fetch_media.call_count == (2 if skip_early_mm_lookup else 1) +@pytest.mark.parametrize( + ("modality", "media"), + [ + pytest.param("image", cherry_pil_image, id="image"), + pytest.param("video", baby_reading_np_ndarrays, id="video"), + ], +) +def test_uuid_cache_eviction_falls_back_to_url( + monkeypatch: pytest.MonkeyPatch, + modality: str, + media: object, +): + renderer = _build_renderer() + mm_processor_cache = renderer.mm_processor_cache + assert mm_processor_cache is not None + + first_lookup = True + + def is_cached_once(_mm_hash: str) -> bool: + nonlocal first_lookup + is_cached = first_lookup + first_lookup = False + return is_cached + + is_cached_item = Mock(side_effect=is_cached_once) + monkeypatch.setattr(mm_processor_cache, "is_cached_item", is_cached_item) + fetch_media = Mock(return_value=media) + monkeypatch.setattr(MediaConnector, f"fetch_{modality}", fetch_media) + + media_url = f"https://example.com/test.{modality}" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": f"Describe this {modality}."}, + { + "type": f"{modality}_url", + f"{modality}_url": {"url": media_url}, + "uuid": f"test-{modality}-uuid", + }, + ], + } + ] + + _, prompts = renderer.render_chat([messages], ChatParams()) + + assert len(prompts[0]["mm_hashes"][modality]) == 1 + assert fetch_media.call_count == 1 + + @pytest.mark.parametrize( ("modality", "media"), [ @@ -149,7 +196,6 @@ def test_early_uuid_lookup_is_disabled_for_unified_vision_chunks( modality: str, media: object, ): - monkeypatch.setenv("VLLM_EARLY_UUID_LOOKUPS", "1") renderer = _build_renderer( model="moonshotai/Kimi-K2.5", trust_remote_code=True, diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 7896db27d51c..811ecc140595 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -439,6 +439,9 @@ class ConversationMessage(TypedDict, total=False): ] _T = TypeVar("_T") _AsyncMultiModalItem: TypeAlias = Callable[[], Awaitable[tuple[object, str | None]]] +MultiModalMediaFallbacks: TypeAlias = dict[ + tuple[str, int], Callable[[], object | Awaitable[object]] +] # Backward compatibility for single item input @@ -563,6 +566,7 @@ def __init__( media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_cache: BaseMultiModalProcessorCache | None = None, skip_early_mm_lookup: bool = False, + media_fallbacks: MultiModalMediaFallbacks | None = None, ): super().__init__() @@ -570,6 +574,7 @@ def __init__( self._media_io_kwargs = media_io_kwargs self._mm_processor_cache = mm_processor_cache self._skip_early_mm_lookup = skip_early_mm_lookup + self._media_fallbacks = media_fallbacks self._items_by_modality = defaultdict[str, list[_T]](list) # Track original modality for each vision_chunk item (image or video) @@ -622,12 +627,11 @@ def is_mm_item_cached( mm_processor_kwargs: Mapping[str, object] | None, ) -> bool: cache = self._mm_processor_cache - if not ( - envs.VLLM_EARLY_UUID_LOOKUPS - and not self._skip_early_mm_lookup - and not self.use_unified_vision_chunk_modality - and uuid is not None - and cache is not None + if ( + self._skip_early_mm_lookup + or self.use_unified_vision_chunk_modality + or uuid is None + or cache is None ): return False @@ -643,6 +647,15 @@ def is_mm_item_cached( ) return cache.is_cached_item(mm_hash) + def add_media_fallback( + self, + modality: Literal["image", "video"], + loader: Callable[[], object | Awaitable[object]], + ) -> None: + if self._media_fallbacks is not None: + item_idx = len(self._items_by_modality[modality]) + self._media_fallbacks[modality, item_idx] = loader + @property def video_processor_name(self) -> str | None: return get_video_processor_cls_name(self.model_config) @@ -1068,12 +1081,18 @@ def parse_prompt_embeds(self, data: str) -> None: self._tracker.add("prompt_embeds", (tensor, None)) self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN) + def _fetch_image(self, image_url: str) -> object: + return self._connector.fetch_image(image_url) + def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: - if self._tracker.is_mm_item_cached( + is_cached = self._tracker.is_mm_item_cached( "image", uuid, self._mm_processor_kwargs - ): - image_url = None - image = self._connector.fetch_image(image_url) if image_url else None + ) + if is_cached and image_url is not None: + self._tracker.add_media_fallback( + "image", partial(self._fetch_image, image_url) + ) + image = None if is_cached or image_url is None else self._fetch_image(image_url) placeholder = self._tracker.add("image", (image, uuid)) self._add_placeholder("image", placeholder) @@ -1160,22 +1179,21 @@ def parse_input_audio( return self.parse_audio(audio_url, uuid) + def _fetch_video(self, video_url: str) -> object: + return self._connector.fetch_video( + video_url=video_url, + video_processor=self._tracker.video_processor_name, + ) + def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: - video_url_to_fetch = ( - None - if self._tracker.is_mm_item_cached( - "video", uuid, self._mm_processor_kwargs - ) - else video_url + is_cached = self._tracker.is_mm_item_cached( + "video", uuid, self._mm_processor_kwargs ) - video = ( - self._connector.fetch_video( - video_url=video_url_to_fetch, - video_processor=self._tracker.video_processor_name, + if is_cached and video_url is not None: + self._tracker.add_media_fallback( + "video", partial(self._fetch_video, video_url) ) - if video_url_to_fetch - else None - ) + video = None if is_cached or video_url is None else self._fetch_video(video_url) placeholder = self._tracker.add("video", (video, uuid)) self._add_placeholder("video", placeholder) @@ -1272,19 +1290,28 @@ async def _load_prompt_embeds_async( tensor = await safe_load_prompt_embeds_async(self.model_config, data_bytes) return tensor, None + async def _fetch_image_async(self, image_url: str) -> object: + return await self._connector.fetch_image_async(image_url) + async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None): - image = ( - await self._connector.fetch_image_async(image_url) if image_url else None - ) + image = await self._fetch_image_async(image_url) if image_url else None return image, uuid def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: - if self._tracker.is_mm_item_cached( + is_cached = self._tracker.is_mm_item_cached( "image", uuid, self._mm_processor_kwargs - ): - image_url = None + ) + if is_cached and image_url is not None: + self._tracker.add_media_fallback( + "image", partial(self._fetch_image_async, image_url) + ) placeholder = self._tracker.add( - "image", partial(self._image_with_uuid_async, image_url, uuid) + "image", + partial( + self._image_with_uuid_async, + None if is_cached else image_url, + uuid, + ), ) self._add_placeholder("image", placeholder) @@ -1400,27 +1427,31 @@ def parse_input_audio( return self.parse_audio(audio_url, uuid) - async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None): - video = ( - await self._connector.fetch_video_async( - video_url, - video_processor=self._tracker.video_processor_name, - ) - if video_url - else None + async def _fetch_video_async(self, video_url: str) -> object: + return await self._connector.fetch_video_async( + video_url, + video_processor=self._tracker.video_processor_name, ) + + async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None): + video = await self._fetch_video_async(video_url) if video_url else None return video, uuid def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: - video_url_to_fetch = ( - None - if self._tracker.is_mm_item_cached( - "video", uuid, self._mm_processor_kwargs - ) - else video_url + is_cached = self._tracker.is_mm_item_cached( + "video", uuid, self._mm_processor_kwargs ) + if is_cached and video_url is not None: + self._tracker.add_media_fallback( + "video", partial(self._fetch_video_async, video_url) + ) placeholder = self._tracker.add( - "video", partial(self._video_with_uuid_async, video_url_to_fetch, uuid) + "video", + partial( + self._video_with_uuid_async, + None if is_cached else video_url, + uuid, + ), ) self._add_placeholder("video", placeholder) @@ -2183,6 +2214,7 @@ def parse_chat_messages( mm_processor_kwargs: dict[str, Any] | None = None, mm_processor_cache: BaseMultiModalProcessorCache | None = None, skip_early_mm_lookup: bool = False, + media_fallbacks: MultiModalMediaFallbacks | None = None, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -2194,6 +2226,7 @@ def parse_chat_messages( media_io_kwargs=media_io_kwargs, mm_processor_cache=mm_processor_cache, skip_early_mm_lookup=skip_early_mm_lookup, + media_fallbacks=media_fallbacks, ) for msg in messages: @@ -2226,6 +2259,7 @@ async def parse_chat_messages_async( mm_processor_kwargs: dict[str, Any] | None = None, mm_processor_cache: BaseMultiModalProcessorCache | None = None, skip_early_mm_lookup: bool = False, + media_fallbacks: MultiModalMediaFallbacks | None = None, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -2237,6 +2271,7 @@ async def parse_chat_messages_async( media_io_kwargs=media_io_kwargs, mm_processor_cache=mm_processor_cache, skip_early_mm_lookup=skip_early_mm_lookup, + media_fallbacks=media_fallbacks, ) for msg in messages: diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index 1deb4d771a17..eef1d59a3f93 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -375,10 +375,7 @@ class ChatCompletionRequest(OpenAIBaseModel): ) skip_early_mm_lookup: bool = Field( default=False, - description=( - "Skip early image and video UUID cache lookups for this request. " - "Only applies when VLLM_EARLY_UUID_LOOKUPS=1." - ), + description="Skip early image and video UUID cache lookups for this request.", ) structured_outputs: StructuredOutputsParams | None = Field( default=None, diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 6246c1365d41..f4fff973de2c 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -242,10 +242,7 @@ class ResponsesRequest(OpenAIBaseModel): ) skip_early_mm_lookup: bool = Field( default=False, - description=( - "Skip early image and video UUID cache lookups for this request. " - "Only applies when VLLM_EARLY_UUID_LOOKUPS=1." - ), + description="Skip early image and video UUID cache lookups for this request.", ) priority: int = Field( default=0, diff --git a/vllm/envs.py b/vllm/envs.py index 5b020164488e..a3ef86d1b6b4 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -74,7 +74,6 @@ VLLM_MEDIA_CACHE: str = "" VLLM_MEDIA_CACHE_MAX_SIZE_MB: int = 5120 VLLM_MEDIA_CACHE_TTL_HOURS: float = 24 - VLLM_EARLY_UUID_LOOKUPS: bool = False VLLM_MEDIA_FETCH_MAX_RETRIES: int = 3 VLLM_MAX_MEDIA_DOWNLOAD_SIZE_MB: int = 256 VLLM_MEDIA_URL_ALLOW_REDIRECTS: bool = True @@ -979,12 +978,6 @@ def _resolve_rust_cli_path() -> str | None: "VLLM_MEDIA_CACHE_TTL_HOURS": lambda: float( os.getenv("VLLM_MEDIA_CACHE_TTL_HOURS", "24") ), - # Whether to probe the multi-modal processor cache before loading image or - # video URLs with UUIDs. Cache hits skip URL loading and decoding. Disabled - # by default. - "VLLM_EARLY_UUID_LOOKUPS": lambda: bool( - int(os.getenv("VLLM_EARLY_UUID_LOOKUPS", "0")) - ), # Maximum number of retries for fetching media (images, audio, video) # from URLs. Each retry quadruples the timeout. Default is 3. "VLLM_MEDIA_FETCH_MAX_RETRIES": lambda: int( @@ -2353,7 +2346,6 @@ def compile_factors() -> dict[str, object]: "VLLM_MEDIA_CACHE", "VLLM_MEDIA_CACHE_MAX_SIZE_MB", "VLLM_MEDIA_CACHE_TTL_HOURS", - "VLLM_EARLY_UUID_LOOKUPS", "VLLM_MEDIA_FETCH_MAX_RETRIES", "VLLM_MEDIA_URL_ALLOW_REDIRECTS", "VLLM_MEDIA_LOADING_THREAD_COUNT", diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 3db820458880..06398b53709e 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -1004,6 +1004,25 @@ def find_mm_placeholders( [`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems]. """ + +@dataclass(frozen=True) +class MissingMultiModalMedia: + modality: str + item_idx: int + + +class MultiModalProcessorCacheMissError(ValueError): + """A processor cache miss for an item whose media data was omitted.""" + + def __init__(self, missing_media: Sequence[MissingMultiModalMedia]) -> None: + self.missing_media = tuple(missing_media) + first = self.missing_media[0] + super().__init__( + f"Cache miss for {first.modality} at index {first.item_idx} " + "but data is not provided." + ) + + MultiModalPromptUpdates = Mapping[str, Sequence[Sequence[ResolvedPromptUpdate]]] """ A collection of prompt updates with a similar structure as @@ -1281,19 +1300,22 @@ def _get_cache_missing_items( } mm_missing_data = {} + missing_media = [] for modality, idxs in mm_missing_idxs.items(): missing_modality_data = [] for idx in idxs: data = mm_data_items[modality][idx] if data is None: - raise ValueError( - f"Cache miss for {modality} at index {idx} " - f"but data is not provided." + missing_media.append( + MissingMultiModalMedia(modality=modality, item_idx=idx) ) else: missing_modality_data.append(data) mm_missing_data[modality] = missing_modality_data + if missing_media: + raise MultiModalProcessorCacheMissError(missing_media) + mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False) return mm_is_cached, mm_missing_items diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index cb8f805be1f4..0c7a5ee97119 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -1,14 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio +import inspect import time import weakref from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack from functools import cached_property -from typing import TYPE_CHECKING, Any, Generic, overload +from typing import TYPE_CHECKING, Any, Generic, cast, overload from typing_extensions import TypeVar @@ -39,6 +40,10 @@ ) from vllm.multimodal.processing import BaseMultiModalProcessor from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs +from vllm.multimodal.processing.processor import ( + MissingMultiModalMedia, + MultiModalProcessorCacheMissError, +) from vllm.multimodal.registry import MultiModalTimingRegistry from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import make_async @@ -69,6 +74,7 @@ _T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike) +_MediaFallbacks = dict[tuple[str, int], Callable[[], object | Awaitable[object]]] class BaseRenderer(ABC, Generic[_T]): @@ -516,7 +522,7 @@ async def render_messages_async( ) -> tuple[list["ConversationMessage"], DictPrompt]: return self.render_messages(messages, params) - # Helpers to enable passing skip_mm_cache to the renderer in some subclasses (e.g. HfRenderer). + # Helpers to pass skip_mm_cache to renderers that support it. def _render_messages_for_chat( self, messages: list["ChatCompletionMessageParam"], @@ -902,6 +908,94 @@ def _process_multimodal( return mm_inputs + @staticmethod + def _replace_missing_mm_data( + mm_data: MultiModalDataDict, + missing_media: Sequence[MissingMultiModalMedia], + loaded_media: Sequence[object], + ) -> MultiModalDataDict: + updated_data = dict(mm_data) + updated_modalities = dict[str, list[object | None]]() + + for missing, media in zip(missing_media, loaded_media): + items = updated_modalities.get(missing.modality) + if items is None: + modality_data = updated_data[missing.modality] + if not isinstance(modality_data, list): + raise RuntimeError( + "Expected URL-backed multi-modal data to be a list, " + f"but got {type(modality_data)} for {missing.modality!r}." + ) + items = list(modality_data) + updated_modalities[missing.modality] = items + updated_data[missing.modality] = items + items[missing.item_idx] = media + + return updated_data + + @staticmethod + def _get_media_fallbacks( + error: MultiModalProcessorCacheMissError, + media_fallbacks: _MediaFallbacks, + ) -> list[Callable[[], object | Awaitable[object]]]: + loaders = [] + for missing in error.missing_media: + loader = media_fallbacks.get((missing.modality, missing.item_idx)) + if loader is None: + raise ValueError( + f"Cache miss for {missing.modality} at index " + f"{missing.item_idx} but data is not provided." + ) from None + loaders.append(loader) + return loaders + + def _load_missing_mm_media( + self, + error: MultiModalProcessorCacheMissError, + media_fallbacks: _MediaFallbacks, + mm_data: MultiModalDataDict, + ) -> MultiModalDataDict: + loaders = self._get_media_fallbacks(error, media_fallbacks) + loaded_media = [loader() for loader in loaders] + if any(inspect.isawaitable(media) for media in loaded_media): + raise RuntimeError("Async media fallback used in synchronous rendering.") + + for missing in error.missing_media: + media_fallbacks.pop((missing.modality, missing.item_idx)) + return self._replace_missing_mm_data(mm_data, error.missing_media, loaded_media) + + async def _load_missing_mm_media_async( + self, + error: MultiModalProcessorCacheMissError, + media_fallbacks: _MediaFallbacks, + mm_data: MultiModalDataDict, + ) -> MultiModalDataDict: + loaders = self._get_media_fallbacks(error, media_fallbacks) + + async def load( + loader: Callable[[], object | Awaitable[object]], + ) -> object: + media = loader() + if inspect.isawaitable(media): + return await media + return media + + loaded_media = await asyncio.gather( + *(load(loader) for loader in loaders), + return_exceptions=True, + ) + for result in loaded_media: + if isinstance(result, BaseException): + raise result + + for missing in error.missing_media: + media_fallbacks.pop((missing.modality, missing.item_idx)) + return self._replace_missing_mm_data( + mm_data, + error.missing_media, + cast(list[object], loaded_media), + ) + def _process_tokens( self, prompt: TokensPrompt, @@ -912,17 +1006,34 @@ def _process_tokens( to the shared thread pool in the async variant. """ prompt_token_ids = prompt["prompt_token_ids"] + media_fallbacks = cast( + _MediaFallbacks, + cast(dict[str, Any], prompt).pop("_mm_media_fallbacks", {}), + ) engine_input: TokensInput | MultiModalInput if multi_modal_data := prompt.get("multi_modal_data"): - engine_input = self._process_multimodal( - prompt_token_ids, - multi_modal_data, - mm_processor_kwargs=prompt.get("mm_processor_kwargs"), - mm_uuids=prompt.get("multi_modal_uuids"), - media_io_kwargs=prompt.get("media_io_kwargs"), - skip_mm_cache=skip_mm_cache, - ) + try: + engine_input = self._process_multimodal( + prompt_token_ids, + multi_modal_data, + mm_processor_kwargs=prompt.get("mm_processor_kwargs"), + mm_uuids=prompt.get("multi_modal_uuids"), + media_io_kwargs=prompt.get("media_io_kwargs"), + skip_mm_cache=skip_mm_cache, + ) + except MultiModalProcessorCacheMissError as error: + multi_modal_data = self._load_missing_mm_media( + error, media_fallbacks, multi_modal_data + ) + engine_input = self._process_multimodal( + prompt_token_ids, + multi_modal_data, + mm_processor_kwargs=prompt.get("mm_processor_kwargs"), + mm_uuids=prompt.get("multi_modal_uuids"), + media_io_kwargs=prompt.get("media_io_kwargs"), + skip_mm_cache=skip_mm_cache, + ) else: engine_input = tokens_input(prompt_token_ids) @@ -975,17 +1086,34 @@ async def _process_tokens_async( skip_mm_cache: bool = False, ) -> TokensInput | MultiModalInput: prompt_token_ids = prompt["prompt_token_ids"] + media_fallbacks = cast( + _MediaFallbacks, + cast(dict[str, Any], prompt).pop("_mm_media_fallbacks", {}), + ) engine_input: TokensInput | MultiModalInput if multi_modal_data := prompt.get("multi_modal_data"): - engine_input = await self._process_multimodal_async( - prompt_token_ids, - multi_modal_data, - mm_processor_kwargs=prompt.get("mm_processor_kwargs"), - mm_uuids=prompt.get("multi_modal_uuids"), - media_io_kwargs=prompt.get("media_io_kwargs"), - skip_mm_cache=skip_mm_cache, - ) + try: + engine_input = await self._process_multimodal_async( + prompt_token_ids, + multi_modal_data, + mm_processor_kwargs=prompt.get("mm_processor_kwargs"), + mm_uuids=prompt.get("multi_modal_uuids"), + media_io_kwargs=prompt.get("media_io_kwargs"), + skip_mm_cache=skip_mm_cache, + ) + except MultiModalProcessorCacheMissError as error: + multi_modal_data = await self._load_missing_mm_media_async( + error, media_fallbacks, multi_modal_data + ) + engine_input = await self._process_multimodal_async( + prompt_token_ids, + multi_modal_data, + mm_processor_kwargs=prompt.get("mm_processor_kwargs"), + mm_uuids=prompt.get("multi_modal_uuids"), + media_io_kwargs=prompt.get("media_io_kwargs"), + skip_mm_cache=skip_mm_cache, + ) else: engine_input = tokens_input(prompt_token_ids) diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 8c2eb66b20de..a1c32ce0265f 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -23,6 +23,7 @@ from vllm.entrypoints.chat_utils import ( PROMPT_EMBEDS_PLACEHOLDER_TOKEN, ChatTemplateResolutionError, + MultiModalMediaFallbacks, load_chat_template, parse_chat_messages, parse_chat_messages_async, @@ -995,6 +996,7 @@ def render_messages( _ensure_prompt_embeds_placeholder_token(tokenizer) ) + media_fallbacks: MultiModalMediaFallbacks = {} conversation, mm_data, mm_uuids = parse_chat_messages( messages, model_config, @@ -1009,6 +1011,7 @@ def render_messages( mm_processor_kwargs=params.mm_processor_kwargs, mm_processor_cache=self.mm_processor_cache, skip_early_mm_lookup=(params.skip_early_mm_lookup or skip_mm_cache), + media_fallbacks=media_fallbacks, ) # prompt_embeds tensors are carried by the tracker through mm_data, @@ -1100,6 +1103,8 @@ def render_messages( prompt["multi_modal_data"] = mm_data if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids + if media_fallbacks: + cast(dict, prompt)["_mm_media_fallbacks"] = media_fallbacks return conversation, prompt @@ -1133,6 +1138,7 @@ async def render_messages_async( _ensure_prompt_embeds_placeholder_token(tokenizer) ) + media_fallbacks: MultiModalMediaFallbacks = {} conversation, mm_data, mm_uuids = await parse_chat_messages_async( messages, model_config, @@ -1147,6 +1153,7 @@ async def render_messages_async( mm_processor_kwargs=params.mm_processor_kwargs, mm_processor_cache=self.mm_processor_cache, skip_early_mm_lookup=(params.skip_early_mm_lookup or skip_mm_cache), + media_fallbacks=media_fallbacks, ) prompt_embeds_tensors: list[torch.Tensor] | None = None @@ -1238,6 +1245,8 @@ async def render_messages_async( prompt["multi_modal_data"] = mm_data if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids + if media_fallbacks: + cast(dict, prompt)["_mm_media_fallbacks"] = media_fallbacks return conversation, prompt From 5deff0ace1c26fc2568915f80af851fffafc923b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kwieci=C5=84ski?= <153385549+Jankwi@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:07:40 +0200 Subject: [PATCH 7/9] [DRAFT]: Simplify lookups MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Kwieciński <153385549+Jankwi@users.noreply.github.com> --- docs/features/multimodal_inputs.md | 21 +- .../test_process_multi_modal_uuids.py | 226 ++++++++++++++++-- vllm/entrypoints/chat_utils.py | 144 ++--------- .../openai/chat_completion/protocol.py | 2 +- vllm/entrypoints/openai/responses/protocol.py | 2 +- vllm/multimodal/processing/inputs.py | 115 ++++----- vllm/multimodal/processing/processor.py | 32 +-- vllm/renderers/base.py | 195 ++------------- vllm/renderers/hf.py | 205 ++++++++++++---- vllm/renderers/params.py | 2 +- 10 files changed, 489 insertions(+), 455 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index adf56cb28699..1f76f538112c 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1409,18 +1409,19 @@ Just like with offline inference, you can skip sending media if you expect cache ``` -#### Early UUID Cache Lookups +#### UUID-Only Rendering -When using the HF renderer, vLLM probes the multi-modal processor cache before -loading an image or video URL with a UUID. Use `skip_early_mm_lookup` to opt out per request. +The HF renderer first attempts to render a conversation without loading media when +all its media items are image/video URLs with UUIDs and at least one URL is supplied. +This is enabled by default. Set `skip_early_mm_lookup=True` to opt out per request. -The first request with an image or video URL and UUID loads and processes the -media as usual. Repeated requests probe multimodal cache first: -* ***On a hit*** vLLM skips loading and decoding the URL. -* ***On a cache miss*** vLLM uses the URL as a fallback and processes the media normally. +- If every media item is in the processor cache, loading and decoding are skipped. +- If any item misses, vLLM retries that conversation once with all the original URLs, + loading the supplied media normally. Other conversations in the batch are not retried. -If some cached items are evicted between the early lookup and multi-modal processing, -vLLM loads the retained URLs and retries processing only the missing items. +This optimization requires the processor cache and is disabled for unified vision +chunks, audio extracted from video, and conversations containing other media types. +An item supplied without a URL must still be cached for the request to succeed. The UUID is authoritative on a cache hit; vLLM does not verify that the URL -still identifies the cached media (see [Multimodal Media UUID Security](../usage/security.md#multimodal-media-uuid-security)) \ No newline at end of file +still identifies the cached media (see [Multimodal Media UUID Security](../usage/security.md#multimodal-media-uuid-security)). diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index 6a1c102c4ea8..2c3b6acc42e1 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -1,8 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +from copy import deepcopy from http import HTTPStatus -from unittest.mock import Mock +from unittest.mock import AsyncMock, Mock import pytest from PIL import Image @@ -13,6 +15,8 @@ from vllm.entrypoints.serve import create_error_response from vllm.multimodal.media import MediaConnector from vllm.multimodal.parse import parse_mm_uuids +from vllm.multimodal.processing.processor import MultiModalProcessorCacheMissError +from vllm.renderers.base import BaseRenderer from vllm.renderers.hf import HfRenderer from vllm.renderers.params import ChatParams from vllm.tokenizers.registry import cached_tokenizer_from_config @@ -85,6 +89,39 @@ def test_text_only_model_mm_data_maps_to_bad_request(): assert error_response.error.code == HTTPStatus.BAD_REQUEST +def _render_chat(renderer, conversations, params, use_async, **kwargs): + if use_async: + return asyncio.run(renderer.render_chat_async(conversations, params, **kwargs)) + return renderer.render_chat(conversations, params, **kwargs) + + +def _mock_media(monkeypatch, modality, media, use_async): + fetch = AsyncMock(return_value=media) if use_async else Mock(return_value=media) + method = f"fetch_{modality}" + ("_async" if use_async else "") + monkeypatch.setattr(MediaConnector, method, fetch) + return fetch + + +def _media_messages(name, *items): + return [ + { + "role": "user", + "content": [{"type": "text", "text": name}] + + [ + { + "type": f"{modality}_url", + f"{modality}_url": { + "url": f"https://example.com/{uuid}.{modality}" + }, + "uuid": uuid, + } + for modality, uuid in items + ], + } + ] + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize( ("modality", "media", "skip_early_mm_lookup"), [ @@ -103,12 +140,12 @@ def test_cached_uuid_skips_url_loading( modality: str, media: object, skip_early_mm_lookup: bool, + use_async: bool, ): renderer = _build_renderer() media_url = f"https://example.com/test.{modality}" media_uuid = f"test-{modality}-uuid" - fetch_media = Mock(return_value=media) - monkeypatch.setattr(MediaConnector, f"fetch_{modality}", fetch_media) + fetch_media = _mock_media(monkeypatch, modality, media, use_async) messages = [ { @@ -125,12 +162,14 @@ def test_cached_uuid_skips_url_loading( ] params = ChatParams(skip_early_mm_lookup=skip_early_mm_lookup) - _, first_prompts = renderer.render_chat([messages], params) - _, second_prompts = renderer.render_chat([messages], params) + _, first_prompts = _render_chat(renderer, [messages], params, use_async) + _, second_prompts = _render_chat(renderer, [messages], params, use_async) first_input = first_prompts[0] second_input = second_prompts[0] assert first_input["mm_hashes"] == second_input["mm_hashes"] + assert first_input["prompt_token_ids"] == second_input["prompt_token_ids"] + assert first_input["mm_placeholders"] == second_input["mm_placeholders"] assert fetch_media.call_count == (2 if skip_early_mm_lookup else 1) @@ -150,16 +189,6 @@ def test_uuid_cache_eviction_falls_back_to_url( mm_processor_cache = renderer.mm_processor_cache assert mm_processor_cache is not None - first_lookup = True - - def is_cached_once(_mm_hash: str) -> bool: - nonlocal first_lookup - is_cached = first_lookup - first_lookup = False - return is_cached - - is_cached_item = Mock(side_effect=is_cached_once) - monkeypatch.setattr(mm_processor_cache, "is_cached_item", is_cached_item) fetch_media = Mock(return_value=media) monkeypatch.setattr(MediaConnector, f"fetch_{modality}", fetch_media) @@ -178,12 +207,170 @@ def is_cached_once(_mm_hash: str) -> bool: } ] + renderer.render_chat([messages], ChatParams()) + renderer.clear_mm_cache() + fetch_media.reset_mock() + _, prompts = renderer.render_chat([messages], ChatParams()) assert len(prompts[0]["mm_hashes"][modality]) == 1 assert fetch_media.call_count == 1 +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_partial_uuid_hit_retries_only_affected_conversation(monkeypatch, use_async): + """A miss reloads all URLs in its conversation without rerendering a batch peer.""" + renderer = _build_renderer() + fetch_image = _mock_media(monkeypatch, "image", cherry_pil_image, use_async) + fetch_video = _mock_media(monkeypatch, "video", baby_reading_np_ndarrays, use_async) + cached = _media_messages("Cached conversation", ("image", "cached-image")) + partial = _media_messages("Partial conversation", ("image", "partial-image")) + params = ChatParams() + _, warm_prompts = _render_chat(renderer, [cached, partial], params, use_async) + partial[0]["content"].append( + _media_messages("", ("video", "missing-video"))[0]["content"][1] + ) + conversations = [cached, partial] + original = deepcopy(conversations) + fetch_image.reset_mock() + fetch_video.reset_mock() + method = "render_messages_async" if use_async else "render_messages" + mock_type = AsyncMock if use_async else Mock + render_messages = mock_type(wraps=getattr(renderer, method)) + monkeypatch.setattr(renderer, method, render_messages) + + _, prompts = _render_chat(renderer, conversations, params, use_async) + + assert [ + call.args[0][0]["content"][0]["text"] for call in render_messages.call_args_list + ].count("Cached conversation") == 1 + assert render_messages.call_count == 3 + fetch_image.assert_called_once_with("https://example.com/partial-image.image") + assert fetch_video.call_count == 1 + assert prompts[0]["prompt_token_ids"] == warm_prompts[0]["prompt_token_ids"] + assert prompts[0]["mm_hashes"] == warm_prompts[0]["mm_hashes"] + assert set(prompts[1]["mm_hashes"]) == {"image", "video"} + assert conversations == original + + fetch_image.reset_mock() + fetch_video.reset_mock() + _, cached_prompts = _render_chat(renderer, conversations, params, use_async) + fetch_image.assert_not_called() + fetch_video.assert_not_called() + assert [p["prompt_token_ids"] for p in cached_prompts] == [ + p["prompt_token_ids"] for p in prompts + ] + + +def test_async_uuid_render_preserves_cache_update_order(monkeypatch): + """Sender cache updates must follow the order of returned conversations.""" + renderer = _build_renderer() + conversations = [ + _media_messages("first", ("image", "shared")), + _media_messages("second", ("image", "shared")), + ] + completed = [] + + async def render(_self, batch, **kwargs): + name = batch[0][0]["content"][0]["text"] + if name == "first": + await asyncio.sleep(0) + completed.append(name) + return [[]], [{"prompt": name}] + + monkeypatch.setattr(BaseRenderer, "render_chat_async", render) + _, prompts = _render_chat(renderer, conversations, ChatParams(), True) + + assert completed == [prompt["prompt"] for prompt in prompts] == ["first", "second"] + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_missing_uuid_loads_all_media(monkeypatch, use_async): + renderer = _build_renderer() + fetch_image = _mock_media(monkeypatch, "image", cherry_pil_image, use_async) + messages = _media_messages("Describe", ("image", "cached-image")) + _render_chat(renderer, [messages], ChatParams(), use_async) + messages[0]["content"].append( + { + "type": "image_url", + "image_url": {"url": "https://example.com/no-uuid.image"}, + } + ) + fetch_image.reset_mock() + + _render_chat(renderer, [messages], ChatParams(), use_async) + + assert fetch_image.call_count == 2 + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +def test_uuid_without_url_still_requires_cache_hit(monkeypatch, use_async): + renderer = _build_renderer() + fetch_image = _mock_media(monkeypatch, "image", cherry_pil_image, use_async) + messages = _media_messages("Describe", ("image", "missing-image")) + messages[0]["content"][1]["image_url"]["url"] = None + + with pytest.raises(MultiModalProcessorCacheMissError, match="data is not provided"): + _render_chat(renderer, [messages], ChatParams(), use_async) + + fetch_image.assert_not_called() + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "case", ["skip-cache", "no-cache", "audio-in-video", "other-media"] +) +def test_ineligible_conversation_renders_original_messages( + monkeypatch, use_async, case +): + renderer = _build_renderer(mm_cache_gb=0 if case == "no-cache" else 4) + messages = _media_messages("Describe", ("image", "image")) + params = ChatParams() + if case == "audio-in-video": + params = ChatParams(mm_processor_kwargs={"use_audio_in_video": True}) + elif case == "other-media": + messages[0]["content"].append( + { + "type": "audio_url", + "audio_url": {"url": "https://example.com/audio.wav"}, + "uuid": "audio", + } + ) + mock_type = AsyncMock if use_async else Mock + render = mock_type(return_value=([[]], [{}])) + method = "render_chat_async" if use_async else "render_chat" + monkeypatch.setattr(BaseRenderer, method, render) + + _render_chat( + renderer, [messages], params, use_async, skip_mm_cache=case == "skip-cache" + ) + + assert render.call_count == 1 + assert render.call_args.args[0][0] is messages + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "error", [ValueError("invalid media"), MultiModalProcessorCacheMissError("missing")] +) +def test_uuid_render_retry_is_bounded_and_only_handles_cache_misses( + monkeypatch, use_async, error +): + renderer = _build_renderer() + messages = _media_messages("Describe", ("image", "image")) + mock_type = AsyncMock if use_async else Mock + render = mock_type(side_effect=error) + method = "render_chat_async" if use_async else "render_chat" + monkeypatch.setattr(BaseRenderer, method, render) + + with pytest.raises(type(error), match=str(error)): + _render_chat(renderer, [messages], ChatParams(), use_async) + + assert render.call_count == ( + 2 if isinstance(error, MultiModalProcessorCacheMissError) else 1 + ) + + @pytest.mark.parametrize( ("modality", "media"), [ @@ -223,10 +410,13 @@ def test_early_uuid_lookup_is_disabled_for_unified_vision_chunks( } ] - conversation, prompt = renderer.render_messages(messages, ChatParams()) + monkeypatch.setattr( + renderer, "process_for_engine", lambda prompt, *args, **kwargs: prompt + ) + conversations, prompts = renderer.render_chat([messages], ChatParams()) - assert len(conversation) == 1 - assert prompt["multi_modal_data"] == {"vision_chunk": [media]} + assert len(conversations[0]) == 1 + assert prompts[0]["multi_modal_data"] == {"vision_chunk": [media]} assert fetch_media.call_count == 1 is_cached_item.assert_not_called() diff --git a/vllm/entrypoints/chat_utils.py b/vllm/entrypoints/chat_utils.py index 67b9ba2897b0..addcc5f76df8 100644 --- a/vllm/entrypoints/chat_utils.py +++ b/vllm/entrypoints/chat_utils.py @@ -6,7 +6,7 @@ import types from abc import ABC, abstractmethod from collections import Counter, defaultdict -from collections.abc import Awaitable, Callable, Iterable, Mapping +from collections.abc import Awaitable, Callable, Iterable from dataclasses import dataclass from functools import cached_property, lru_cache, partial from itertools import accumulate @@ -57,7 +57,6 @@ from vllm.logger import init_logger from vllm.model_executor.models import SupportsMultiModal from vllm.multimodal import MULTIMODAL_REGISTRY -from vllm.multimodal.cache import BaseMultiModalProcessorCache from vllm.multimodal.inputs import ( MultiModalBatchedField, MultiModalFlatField, @@ -67,7 +66,7 @@ VisionChunkVideo, ) from vllm.multimodal.media import MEDIA_CONNECTOR_REGISTRY, MediaConnector -from vllm.multimodal.processing import BaseMultiModalProcessor, ProcessorInputs +from vllm.multimodal.processing import BaseMultiModalProcessor from vllm.renderers.embed_utils import ( safe_load_prompt_embeds, safe_load_prompt_embeds_async, @@ -442,9 +441,6 @@ class ConversationMessage(TypedDict, total=False): ] _T = TypeVar("_T") _AsyncMultiModalItem: TypeAlias = Callable[[], Awaitable[tuple[object, str | None]]] -MultiModalMediaFallbacks: TypeAlias = dict[ - tuple[str, int], Callable[[], object | Awaitable[object]] -] # Backward compatibility for single item input @@ -600,17 +596,11 @@ def __init__( self, model_config: ModelConfig, media_io_kwargs: dict[str, dict[str, Any]] | None = None, - mm_processor_cache: BaseMultiModalProcessorCache | None = None, - skip_early_mm_lookup: bool = False, - media_fallbacks: MultiModalMediaFallbacks | None = None, ): super().__init__() self._model_config = model_config self._media_io_kwargs = media_io_kwargs - self._mm_processor_cache = mm_processor_cache - self._skip_early_mm_lookup = skip_early_mm_lookup - self._media_fallbacks = media_fallbacks self._items_by_modality = defaultdict[str, list[_T]](list) # Track original modality for each vision_chunk item (image or video) @@ -656,42 +646,6 @@ def mm_registry(self): def mm_processor(self): return self.mm_registry.create_processor(self.model_config) - def is_mm_item_cached( - self, - modality: str, - uuid: str | None, - mm_processor_kwargs: Mapping[str, object] | None, - ) -> bool: - cache = self._mm_processor_cache - if ( - self._skip_early_mm_lookup - or self.use_unified_vision_chunk_modality - or uuid is None - or cache is None - ): - return False - - mm_config = self.model_config.get_multimodal_config() - mm_hash = ProcessorInputs.get_mm_item_hash( - modality, - None, - uuid, - self.model_config.model, - mm_config.mm_hasher_algorithm, - mm_processor_kwargs or {}, - self.media_io_kwargs or {}, - ) - return cache.is_cached_item(mm_hash) - - def add_media_fallback( - self, - modality: Literal["image", "video"], - loader: Callable[[], object | Awaitable[object]], - ) -> None: - if self._media_fallbacks is not None: - item_idx = len(self._items_by_modality[modality]) - self._media_fallbacks[modality, item_idx] = loader - @property def video_processor_name(self) -> str | None: return get_video_processor_cls_name(self.model_config) @@ -1117,18 +1071,8 @@ def parse_prompt_embeds(self, data: str) -> None: self._tracker.add("prompt_embeds", (tensor, None)) self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN) - def _fetch_image(self, image_url: str) -> object: - return self._connector.fetch_image(image_url) - def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: - is_cached = self._tracker.is_mm_item_cached( - "image", uuid, self._mm_processor_kwargs - ) - if is_cached and image_url is not None: - self._tracker.add_media_fallback( - "image", partial(self._fetch_image, image_url) - ) - image = None if is_cached or image_url is None else self._fetch_image(image_url) + image = self._connector.fetch_image(image_url) if image_url else None placeholder = self._tracker.add("image", (image, uuid)) self._add_placeholder("image", placeholder) @@ -1215,21 +1159,15 @@ def parse_input_audio( return self.parse_audio(audio_url, uuid) - def _fetch_video(self, video_url: str) -> object: - return self._connector.fetch_video( - video_url=video_url, - video_processor=self._tracker.video_processor_name, - ) - def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: - is_cached = self._tracker.is_mm_item_cached( - "video", uuid, self._mm_processor_kwargs - ) - if is_cached and video_url is not None: - self._tracker.add_media_fallback( - "video", partial(self._fetch_video, video_url) + video = ( + self._connector.fetch_video( + video_url=video_url, + video_processor=self._tracker.video_processor_name, ) - video = None if is_cached or video_url is None else self._fetch_video(video_url) + if video_url + else None + ) placeholder = self._tracker.add("video", (video, uuid)) self._add_placeholder("video", placeholder) @@ -1326,28 +1264,15 @@ async def _load_prompt_embeds_async( tensor = await safe_load_prompt_embeds_async(self.model_config, data_bytes) return tensor, None - async def _fetch_image_async(self, image_url: str) -> object: - return await self._connector.fetch_image_async(image_url) - async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None): - image = await self._fetch_image_async(image_url) if image_url else None + image = ( + await self._connector.fetch_image_async(image_url) if image_url else None + ) return image, uuid def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: - is_cached = self._tracker.is_mm_item_cached( - "image", uuid, self._mm_processor_kwargs - ) - if is_cached and image_url is not None: - self._tracker.add_media_fallback( - "image", partial(self._fetch_image_async, image_url) - ) placeholder = self._tracker.add( - "image", - partial( - self._image_with_uuid_async, - None if is_cached else image_url, - uuid, - ), + "image", partial(self._image_with_uuid_async, image_url, uuid) ) self._add_placeholder("image", placeholder) @@ -1455,31 +1380,20 @@ def parse_input_audio( return self.parse_audio(audio_url, uuid) - async def _fetch_video_async(self, video_url: str) -> object: - return await self._connector.fetch_video_async( - video_url, - video_processor=self._tracker.video_processor_name, - ) - async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None): - video = await self._fetch_video_async(video_url) if video_url else None + video = ( + await self._connector.fetch_video_async( + video_url, + video_processor=self._tracker.video_processor_name, + ) + if video_url + else None + ) return video, uuid def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: - is_cached = self._tracker.is_mm_item_cached( - "video", uuid, self._mm_processor_kwargs - ) - if is_cached and video_url is not None: - self._tracker.add_media_fallback( - "video", partial(self._fetch_video_async, video_url) - ) placeholder = self._tracker.add( - "video", - partial( - self._video_with_uuid_async, - None if is_cached else video_url, - uuid, - ), + "video", partial(self._video_with_uuid_async, video_url, uuid) ) self._add_placeholder("video", placeholder) @@ -2247,9 +2161,6 @@ def parse_chat_messages( content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, - mm_processor_cache: BaseMultiModalProcessorCache | None = None, - skip_early_mm_lookup: bool = False, - media_fallbacks: MultiModalMediaFallbacks | None = None, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -2259,9 +2170,6 @@ def parse_chat_messages( mm_tracker = MultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, - mm_processor_cache=mm_processor_cache, - skip_early_mm_lookup=skip_early_mm_lookup, - media_fallbacks=media_fallbacks, ) for msg in messages: @@ -2292,9 +2200,6 @@ async def parse_chat_messages_async( content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, - mm_processor_cache: BaseMultiModalProcessorCache | None = None, - skip_early_mm_lookup: bool = False, - media_fallbacks: MultiModalMediaFallbacks | None = None, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -2304,9 +2209,6 @@ async def parse_chat_messages_async( mm_tracker = AsyncMultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, - mm_processor_cache=mm_processor_cache, - skip_early_mm_lookup=skip_early_mm_lookup, - media_fallbacks=media_fallbacks, ) for msg in messages: diff --git a/vllm/entrypoints/openai/chat_completion/protocol.py b/vllm/entrypoints/openai/chat_completion/protocol.py index fd96b03feaa5..53b474fffab3 100644 --- a/vllm/entrypoints/openai/chat_completion/protocol.py +++ b/vllm/entrypoints/openai/chat_completion/protocol.py @@ -376,7 +376,7 @@ class ChatCompletionRequest(OpenAIBaseModel): ) skip_early_mm_lookup: bool = Field( default=False, - description="Skip early image and video UUID cache lookups for this request.", + description="Skip the UUID-only rendering attempt for this request.", ) structured_outputs: StructuredOutputsParams | None = Field( default=None, diff --git a/vllm/entrypoints/openai/responses/protocol.py b/vllm/entrypoints/openai/responses/protocol.py index 1987ff10c2e8..0c60d220f79a 100644 --- a/vllm/entrypoints/openai/responses/protocol.py +++ b/vllm/entrypoints/openai/responses/protocol.py @@ -243,7 +243,7 @@ class ResponsesRequest(OpenAIBaseModel): ) skip_early_mm_lookup: bool = Field( default=False, - description="Skip early image and video UUID cache lookups for this request.", + description="Skip the UUID-only rendering attempt for this request.", ) priority: int = Field( default=0, diff --git a/vllm/multimodal/processing/inputs.py b/vllm/multimodal/processing/inputs.py index e277cbcc724b..56750c3087cb 100644 --- a/vllm/multimodal/processing/inputs.py +++ b/vllm/multimodal/processing/inputs.py @@ -29,59 +29,6 @@ class ProcessorInputs: hf_processor_mm_kwargs: Mapping[str, object] = field(default_factory=dict) media_io_kwargs: Mapping[str, Mapping[str, object]] = field(default_factory=dict) - @staticmethod - def get_mm_item_hash( - modality: str, - item: object, - uuid_item: str | None, - model_id: str, - hash_algorithm: MMHasherAlgorithm, - hf_processor_mm_kwargs: Mapping[str, object], - media_io_kwargs: Mapping[str, Mapping[str, object]], - ) -> str: - # Each cache entry represents one item from one modality. Keep - # modality-scoped options out of unrelated entries, while retaining - # flat processor kwargs because they are shared by HF. - if modality in _HF_MODALITY_PROCESSOR_KWARGS: - modality_media_io_kwargs = media_io_kwargs.get(modality, {}) - scoped_media_io_kwargs = ( - {modality: modality_media_io_kwargs} - if modality_media_io_kwargs - else {} - ) - scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality] - mm_processor_kwargs = { - key: value - for key, value in hf_processor_mm_kwargs.items() - if key not in _HF_MODALITY_PROCESSOR_KWARGS.values() - or key == scoped_key - } - else: - # Unified modalities such as ``vision_chunk`` can contain items - # originating from multiple media modalities. There is no per-item - # origin metadata here, so preserve all factors. - scoped_media_io_kwargs = dict(media_io_kwargs) - mm_processor_kwargs = dict(hf_processor_mm_kwargs) - - hash_factors = { - "media_io_kwargs": scoped_media_io_kwargs, - "mm_processor_kwargs": mm_processor_kwargs, - } - hash_factors = {key: value for key, value in hash_factors.items() if value} - - # Even if a UUID is provided, model output depends on the current - # modality's hash factors, so they are taken into account. - if uuid_item is None or hash_factors: - item = uuid_item if uuid_item is not None else item - return MultiModalHasher.hash_kwargs( - hash_algorithm, - model_id=model_id, - **{modality: item}, - **hash_factors, - ) - - return uuid_item - def get_mm_hashes( self, model_id: str, @@ -91,7 +38,40 @@ def get_mm_hashes( mm_uuid_items = self.mm_uuid_items or {} mm_hashes = dict[str, list[str]]() + hasher = MultiModalHasher + for modality, data_items in mm_data_items.items(): + # Each cache entry represents one item from one modality. Keep + # modality-scoped options out of unrelated entries, while + # retaining flat processor kwargs because they are shared by HF. + if modality in _HF_MODALITY_PROCESSOR_KWARGS: + modality_media_io_kwargs = self.media_io_kwargs.get(modality, {}) + media_io_kwargs = ( + {modality: modality_media_io_kwargs} + if modality_media_io_kwargs + else {} + ) + scoped_key = _HF_MODALITY_PROCESSOR_KWARGS[modality] + mm_processor_kwargs = { + key: value + for key, value in self.hf_processor_mm_kwargs.items() + if key not in _HF_MODALITY_PROCESSOR_KWARGS.values() + or key == scoped_key + } + else: + # Unified modalities such as ``vision_chunk`` can contain + # items originating from multiple media modalities. There is + # no per-item origin metadata here, so preserve all factors. + media_io_kwargs = dict(self.media_io_kwargs) + mm_processor_kwargs = dict(self.hf_processor_mm_kwargs) + + hash_factors = { + "media_io_kwargs": media_io_kwargs, + "mm_processor_kwargs": mm_processor_kwargs, + } + hash_factors = {key: value for key, value in hash_factors.items() if value} + has_hash_factors = bool(hash_factors) + uuid_items = ( mm_uuid_items[modality] if modality in mm_uuid_items @@ -101,17 +81,26 @@ def get_mm_hashes( # For None entries, compute a hash; otherwise, use provided ID. hashes: list[str] = [] for i, item in enumerate(data_items.get_all_items_for_hash()): - hashes.append( - self.get_mm_item_hash( - modality, - item, - uuid_items[i], - model_id, - hash_algorithm, - self.hf_processor_mm_kwargs, - self.media_io_kwargs, + uuid_item = uuid_items[i] + + # NOTE: Even if a uuid_item is provided, model output depends + # on the current modality's hash factors, so they are taken + # into account. + if uuid_item is None or has_hash_factors: + # NOTE: use provided hash string to hash with kwargs + # if available for better performance. + item = uuid_item if uuid_item is not None else item + hashes.append( + hasher.hash_kwargs( + hash_algorithm, + model_id=model_id, + **{modality: item}, + **hash_factors, + ) ) - ) + else: + # If there are no extra kwargs, use the client-provided UUID. + hashes.append(uuid_item) mm_hashes[modality] = hashes diff --git a/vllm/multimodal/processing/processor.py b/vllm/multimodal/processing/processor.py index 06398b53709e..241ff1787b51 100644 --- a/vllm/multimodal/processing/processor.py +++ b/vllm/multimodal/processing/processor.py @@ -998,31 +998,16 @@ def find_mm_placeholders( return dict(full_groupby_modality(it)) +class MultiModalProcessorCacheMissError(ValueError): + """A processor cache miss for an item whose media data was omitted.""" + + MultiModalIsCached = dict[str, list[bool]] """ A collection of the `is_cached` flag for each item, with a similar structure as [`MultiModalKwargsItems`][vllm.multimodal.inputs.MultiModalKwargsItems]. """ - -@dataclass(frozen=True) -class MissingMultiModalMedia: - modality: str - item_idx: int - - -class MultiModalProcessorCacheMissError(ValueError): - """A processor cache miss for an item whose media data was omitted.""" - - def __init__(self, missing_media: Sequence[MissingMultiModalMedia]) -> None: - self.missing_media = tuple(missing_media) - first = self.missing_media[0] - super().__init__( - f"Cache miss for {first.modality} at index {first.item_idx} " - "but data is not provided." - ) - - MultiModalPromptUpdates = Mapping[str, Sequence[Sequence[ResolvedPromptUpdate]]] """ A collection of prompt updates with a similar structure as @@ -1300,22 +1285,19 @@ def _get_cache_missing_items( } mm_missing_data = {} - missing_media = [] for modality, idxs in mm_missing_idxs.items(): missing_modality_data = [] for idx in idxs: data = mm_data_items[modality][idx] if data is None: - missing_media.append( - MissingMultiModalMedia(modality=modality, item_idx=idx) + raise MultiModalProcessorCacheMissError( + f"Cache miss for {modality} at index {idx} " + f"but data is not provided." ) else: missing_modality_data.append(data) mm_missing_data[modality] = missing_modality_data - if missing_media: - raise MultiModalProcessorCacheMissError(missing_media) - mm_missing_items = self.info.parse_mm_data(mm_missing_data, validate=False) return mm_is_cached, mm_missing_items diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 0c7a5ee97119..0fd1b7610bae 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -1,15 +1,14 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio -import inspect import time import weakref from abc import ABC, abstractmethod -from collections.abc import Awaitable, Callable, Mapping, Sequence +from collections.abc import Mapping, Sequence from concurrent.futures import Future, ThreadPoolExecutor from contextlib import ExitStack from functools import cached_property -from typing import TYPE_CHECKING, Any, Generic, cast, overload +from typing import TYPE_CHECKING, Any, Generic, overload from typing_extensions import TypeVar @@ -40,10 +39,6 @@ ) from vllm.multimodal.processing import BaseMultiModalProcessor from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs -from vllm.multimodal.processing.processor import ( - MissingMultiModalMedia, - MultiModalProcessorCacheMissError, -) from vllm.multimodal.registry import MultiModalTimingRegistry from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import make_async @@ -74,7 +69,6 @@ _T = TypeVar("_T", bound=TokenizerLike, default=TokenizerLike) -_MediaFallbacks = dict[tuple[str, int], Callable[[], object | Awaitable[object]]] class BaseRenderer(ABC, Generic[_T]): @@ -522,25 +516,6 @@ async def render_messages_async( ) -> tuple[list["ConversationMessage"], DictPrompt]: return self.render_messages(messages, params) - # Helpers to pass skip_mm_cache to renderers that support it. - def _render_messages_for_chat( - self, - messages: list["ChatCompletionMessageParam"], - params: ChatParams, - *, - skip_mm_cache: bool, - ) -> tuple[list["ConversationMessage"], DictPrompt]: - return self.render_messages(messages, params) - - async def _render_messages_for_chat_async( - self, - messages: list["ChatCompletionMessageParam"], - params: ChatParams, - *, - skip_mm_cache: bool, - ) -> tuple[list["ConversationMessage"], DictPrompt]: - return await self.render_messages_async(messages, params) - # Step 2: Tokenize prompts if necessary def _can_produce_offsets(self) -> bool: """Whether this renderer's tokenizer can emit char-level offsets. @@ -908,94 +883,6 @@ def _process_multimodal( return mm_inputs - @staticmethod - def _replace_missing_mm_data( - mm_data: MultiModalDataDict, - missing_media: Sequence[MissingMultiModalMedia], - loaded_media: Sequence[object], - ) -> MultiModalDataDict: - updated_data = dict(mm_data) - updated_modalities = dict[str, list[object | None]]() - - for missing, media in zip(missing_media, loaded_media): - items = updated_modalities.get(missing.modality) - if items is None: - modality_data = updated_data[missing.modality] - if not isinstance(modality_data, list): - raise RuntimeError( - "Expected URL-backed multi-modal data to be a list, " - f"but got {type(modality_data)} for {missing.modality!r}." - ) - items = list(modality_data) - updated_modalities[missing.modality] = items - updated_data[missing.modality] = items - items[missing.item_idx] = media - - return updated_data - - @staticmethod - def _get_media_fallbacks( - error: MultiModalProcessorCacheMissError, - media_fallbacks: _MediaFallbacks, - ) -> list[Callable[[], object | Awaitable[object]]]: - loaders = [] - for missing in error.missing_media: - loader = media_fallbacks.get((missing.modality, missing.item_idx)) - if loader is None: - raise ValueError( - f"Cache miss for {missing.modality} at index " - f"{missing.item_idx} but data is not provided." - ) from None - loaders.append(loader) - return loaders - - def _load_missing_mm_media( - self, - error: MultiModalProcessorCacheMissError, - media_fallbacks: _MediaFallbacks, - mm_data: MultiModalDataDict, - ) -> MultiModalDataDict: - loaders = self._get_media_fallbacks(error, media_fallbacks) - loaded_media = [loader() for loader in loaders] - if any(inspect.isawaitable(media) for media in loaded_media): - raise RuntimeError("Async media fallback used in synchronous rendering.") - - for missing in error.missing_media: - media_fallbacks.pop((missing.modality, missing.item_idx)) - return self._replace_missing_mm_data(mm_data, error.missing_media, loaded_media) - - async def _load_missing_mm_media_async( - self, - error: MultiModalProcessorCacheMissError, - media_fallbacks: _MediaFallbacks, - mm_data: MultiModalDataDict, - ) -> MultiModalDataDict: - loaders = self._get_media_fallbacks(error, media_fallbacks) - - async def load( - loader: Callable[[], object | Awaitable[object]], - ) -> object: - media = loader() - if inspect.isawaitable(media): - return await media - return media - - loaded_media = await asyncio.gather( - *(load(loader) for loader in loaders), - return_exceptions=True, - ) - for result in loaded_media: - if isinstance(result, BaseException): - raise result - - for missing in error.missing_media: - media_fallbacks.pop((missing.modality, missing.item_idx)) - return self._replace_missing_mm_data( - mm_data, - error.missing_media, - cast(list[object], loaded_media), - ) - def _process_tokens( self, prompt: TokensPrompt, @@ -1006,34 +893,17 @@ def _process_tokens( to the shared thread pool in the async variant. """ prompt_token_ids = prompt["prompt_token_ids"] - media_fallbacks = cast( - _MediaFallbacks, - cast(dict[str, Any], prompt).pop("_mm_media_fallbacks", {}), - ) engine_input: TokensInput | MultiModalInput if multi_modal_data := prompt.get("multi_modal_data"): - try: - engine_input = self._process_multimodal( - prompt_token_ids, - multi_modal_data, - mm_processor_kwargs=prompt.get("mm_processor_kwargs"), - mm_uuids=prompt.get("multi_modal_uuids"), - media_io_kwargs=prompt.get("media_io_kwargs"), - skip_mm_cache=skip_mm_cache, - ) - except MultiModalProcessorCacheMissError as error: - multi_modal_data = self._load_missing_mm_media( - error, media_fallbacks, multi_modal_data - ) - engine_input = self._process_multimodal( - prompt_token_ids, - multi_modal_data, - mm_processor_kwargs=prompt.get("mm_processor_kwargs"), - mm_uuids=prompt.get("multi_modal_uuids"), - media_io_kwargs=prompt.get("media_io_kwargs"), - skip_mm_cache=skip_mm_cache, - ) + engine_input = self._process_multimodal( + prompt_token_ids, + multi_modal_data, + mm_processor_kwargs=prompt.get("mm_processor_kwargs"), + mm_uuids=prompt.get("multi_modal_uuids"), + media_io_kwargs=prompt.get("media_io_kwargs"), + skip_mm_cache=skip_mm_cache, + ) else: engine_input = tokens_input(prompt_token_ids) @@ -1086,34 +956,17 @@ async def _process_tokens_async( skip_mm_cache: bool = False, ) -> TokensInput | MultiModalInput: prompt_token_ids = prompt["prompt_token_ids"] - media_fallbacks = cast( - _MediaFallbacks, - cast(dict[str, Any], prompt).pop("_mm_media_fallbacks", {}), - ) engine_input: TokensInput | MultiModalInput if multi_modal_data := prompt.get("multi_modal_data"): - try: - engine_input = await self._process_multimodal_async( - prompt_token_ids, - multi_modal_data, - mm_processor_kwargs=prompt.get("mm_processor_kwargs"), - mm_uuids=prompt.get("multi_modal_uuids"), - media_io_kwargs=prompt.get("media_io_kwargs"), - skip_mm_cache=skip_mm_cache, - ) - except MultiModalProcessorCacheMissError as error: - multi_modal_data = await self._load_missing_mm_media_async( - error, media_fallbacks, multi_modal_data - ) - engine_input = await self._process_multimodal_async( - prompt_token_ids, - multi_modal_data, - mm_processor_kwargs=prompt.get("mm_processor_kwargs"), - mm_uuids=prompt.get("multi_modal_uuids"), - media_io_kwargs=prompt.get("media_io_kwargs"), - skip_mm_cache=skip_mm_cache, - ) + engine_input = await self._process_multimodal_async( + prompt_token_ids, + multi_modal_data, + mm_processor_kwargs=prompt.get("mm_processor_kwargs"), + mm_uuids=prompt.get("multi_modal_uuids"), + media_io_kwargs=prompt.get("media_io_kwargs"), + skip_mm_cache=skip_mm_cache, + ) else: engine_input = tokens_input(prompt_token_ids) @@ -1314,11 +1167,7 @@ def render_chat( tok_params = self.default_chat_tok_params rendered = [ - self._render_messages_for_chat( - conversation, - chat_params, - skip_mm_cache=skip_mm_cache, - ) + self.render_messages(conversation, chat_params) for conversation in conversations ] @@ -1356,11 +1205,7 @@ async def render_chat_async( tok_params = self.default_chat_tok_params rendered = [ - self._render_messages_for_chat_async( - conversation, - chat_params, - skip_mm_cache=skip_mm_cache, - ) + self.render_messages_async(conversation, chat_params) for conversation in conversations ] diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 93ee68cdee39..e97574e4b966 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -8,7 +8,7 @@ import weakref from collections import defaultdict, deque from collections.abc import Mapping, Sequence -from functools import lru_cache +from functools import lru_cache, partial from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import jinja2 @@ -23,7 +23,6 @@ from vllm.entrypoints.chat_utils import ( PROMPT_EMBEDS_PLACEHOLDER_TOKEN, ChatTemplateResolutionError, - MultiModalMediaFallbacks, load_chat_template, parse_chat_messages, parse_chat_messages_async, @@ -41,6 +40,7 @@ PlaceholderRange, ) from vllm.multimodal.processing.processor import ( + MultiModalProcessorCacheMissError, PromptReplacement, apply_token_matches, find_mm_placeholders, @@ -72,7 +72,7 @@ ) from .inputs import DictPrompt - from .params import ChatParams + from .params import ChatParams, TokenizeParams logger = init_logger(__name__) @@ -1018,26 +1018,179 @@ def _can_produce_offsets(self) -> bool: # expose offset_mapping. return self.tokenizer is not None and self.tokenizer.is_fast - @override - def _render_messages_for_chat( + def _get_uuid_only_messages( self, messages: list[ChatCompletionMessageParam], params: ChatParams, + ) -> list[ChatCompletionMessageParam] | None: + """Omit URLs only when every media item can be looked up by UUID.""" + if ( + params.skip_early_mm_lookup + or self.mm_processor_cache is None + or self.use_unified_vision_chunk + ): + return None + + mm_config = self.model_config.get_multimodal_config() + processor_kwargs = mm_config.merge_mm_processor_kwargs( + params.mm_processor_kwargs or {} + ) + if processor_kwargs.get("use_audio_in_video", False): + return None + + uuid_only_messages = [] + has_url = False + for message in messages: + content = message.get("content") + if content is None or isinstance(content, str): + uuid_only_messages.append(message) + continue + if not isinstance(content, list): + return None + + uuid_only_content: list[Any] = [] + for part in content: + if isinstance(part, str) or part.get("type") in ( + "text", + "input_text", + "output_text", + "refusal", + "thinking", + "tool_reference", + ): + uuid_only_content.append(part) + continue + if part.get("type") not in ( + None, + "image_url", + "input_image", + "video_url", + ): + return None + url_key = next( + (key for key in ("image_url", "video_url") if key in part), None + ) + if url_key is None or part.get("uuid") is None: + return None + + media = cast(dict[str, Any], part)[url_key] + url = media.get("url") if isinstance(media, dict) else media + has_url |= url is not None + uuid_only_content.append( + { + **part, + url_key: {**media, "url": None} + if isinstance(media, dict) + else None, + } + ) + + uuid_only_messages.append( + cast( + "ChatCompletionMessageParam", + {**message, "content": uuid_only_content}, + ) + ) + + return uuid_only_messages if has_url else None + + @override + def render_chat( + self, + conversations: Sequence[list[ChatCompletionMessageParam]], + chat_params: ChatParams, + tok_params: TokenizeParams | None = None, *, - skip_mm_cache: bool, - ) -> tuple[list[ConversationMessage], DictPrompt]: - return self.render_messages( - messages, - params, + prompt_extras: dict[str, Any] | None = None, + skip_mm_cache: bool = False, + ): + render = partial( + super().render_chat, + chat_params=chat_params, + tok_params=tok_params, + prompt_extras=prompt_extras, skip_mm_cache=skip_mm_cache, ) + uuid_only_conversations = [ + None + if skip_mm_cache + else self._get_uuid_only_messages(messages, chat_params) + for messages in conversations + ] + if all(messages is None for messages in uuid_only_conversations): + return render(conversations) + + def render_one( + messages: list[ChatCompletionMessageParam], + uuid_only: list[ChatCompletionMessageParam] | None, + ): + if uuid_only is not None: + try: + return render([uuid_only]) + except MultiModalProcessorCacheMissError: + pass + return render([messages]) + + rendered = [ + render_one(messages, uuid_only) + for messages, uuid_only in zip(conversations, uuid_only_conversations) + ] + return [result[0][0] for result in rendered], [ + result[1][0] for result in rendered + ] + + @override + async def render_chat_async( + self, + conversations: Sequence[list[ChatCompletionMessageParam]], + chat_params: ChatParams, + tok_params: TokenizeParams | None = None, + *, + prompt_extras: dict[str, Any] | None = None, + skip_mm_cache: bool = False, + ): + render = partial( + super().render_chat_async, + chat_params=chat_params, + tok_params=tok_params, + prompt_extras=prompt_extras, + skip_mm_cache=skip_mm_cache, + ) + + uuid_only_conversations = [ + None + if skip_mm_cache + else self._get_uuid_only_messages(messages, chat_params) + for messages in conversations + ] + if all(messages is None for messages in uuid_only_conversations): + return await render(conversations) + + async def render_one( + messages: list[ChatCompletionMessageParam], + uuid_only: list[ChatCompletionMessageParam] | None, + ): + if uuid_only is not None: + try: + return await render([uuid_only]) + except MultiModalProcessorCacheMissError: + pass + return await render([messages]) + + # Sender-cache updates must follow the order of returned conversations. + rendered = [ + await render_one(messages, uuid_only) + for messages, uuid_only in zip(conversations, uuid_only_conversations) + ] + return [result[0][0] for result in rendered], [ + result[1][0] for result in rendered + ] + def render_messages( self, messages: list[ChatCompletionMessageParam], params: ChatParams, - *, - skip_mm_cache: bool = False, ) -> tuple[list[ConversationMessage], DictPrompt]: model_config = self.model_config tokenizer = self.get_tokenizer() @@ -1048,7 +1201,6 @@ def render_messages( _ensure_prompt_embeds_placeholder_token(tokenizer) ) - media_fallbacks: MultiModalMediaFallbacks = {} conversation, mm_data, mm_uuids = parse_chat_messages( messages, model_config, @@ -1061,9 +1213,6 @@ def render_messages( ), media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, - mm_processor_cache=self.mm_processor_cache, - skip_early_mm_lookup=(params.skip_early_mm_lookup or skip_mm_cache), - media_fallbacks=media_fallbacks, ) # prompt_embeds tensors are carried by the tracker through mm_data, @@ -1155,31 +1304,13 @@ def render_messages( prompt["multi_modal_data"] = mm_data if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - if media_fallbacks: - cast(dict, prompt)["_mm_media_fallbacks"] = media_fallbacks return conversation, prompt - @override - async def _render_messages_for_chat_async( - self, - messages: list[ChatCompletionMessageParam], - params: ChatParams, - *, - skip_mm_cache: bool, - ) -> tuple[list[ConversationMessage], DictPrompt]: - return await self.render_messages_async( - messages, - params, - skip_mm_cache=skip_mm_cache, - ) - async def render_messages_async( self, messages: list[ChatCompletionMessageParam], params: ChatParams, - *, - skip_mm_cache: bool = False, ) -> tuple[list[ConversationMessage], DictPrompt]: model_config = self.model_config tokenizer = self.get_tokenizer() @@ -1190,7 +1321,6 @@ async def render_messages_async( _ensure_prompt_embeds_placeholder_token(tokenizer) ) - media_fallbacks: MultiModalMediaFallbacks = {} conversation, mm_data, mm_uuids = await parse_chat_messages_async( messages, model_config, @@ -1203,9 +1333,6 @@ async def render_messages_async( ), media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, - mm_processor_cache=self.mm_processor_cache, - skip_early_mm_lookup=(params.skip_early_mm_lookup or skip_mm_cache), - media_fallbacks=media_fallbacks, ) prompt_embeds_tensors: list[torch.Tensor] | None = None @@ -1297,8 +1424,6 @@ async def render_messages_async( prompt["multi_modal_data"] = mm_data if mm_uuids is not None: prompt["multi_modal_uuids"] = mm_uuids - if media_fallbacks: - cast(dict, prompt)["_mm_media_fallbacks"] = media_fallbacks return conversation, prompt diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index efc90bb230e6..245087af56c7 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -100,7 +100,7 @@ class ChatParams: """The kwargs to pass to the multi-modal processor.""" skip_early_mm_lookup: bool = False - """Skip early UUID-based multi-modal cache lookups.""" + """Skip the UUID-only rendering attempt.""" return_assistant_tokens_mask: bool = False """Request a per-token assistant mask from apply_chat_template.""" From 1348d4e28ab3251e9d9edc10299655a285106c17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kwieci=C5=84ski?= <153385549+Jankwi@users.noreply.github.com> Date: Fri, 23 Oct 2026 11:29:53 +0200 Subject: [PATCH 8/9] Complete redesign MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Kwieciński <153385549+Jankwi@users.noreply.github.com> --- docs/features/multimodal_inputs.md | 13 +- tests/renderers/chat_utils/test_audio.py | 44 +++ .../test_process_multi_modal_uuids.py | 310 +++++++----------- vllm/renderers/base.py | 223 ++++++++++--- vllm/renderers/chat_utils/__init__.py | 5 +- vllm/renderers/chat_utils/chat_message.py | 51 ++- vllm/renderers/chat_utils/content_parser.py | 31 +- vllm/renderers/chat_utils/item_tracker.py | 6 + vllm/renderers/hf.py | 206 +++--------- vllm/renderers/params.py | 2 +- 10 files changed, 465 insertions(+), 426 deletions(-) diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index 13e6a7d77224..b95ac91260a4 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1420,16 +1420,17 @@ Just like with offline inference, you can skip sending media if you expect cache #### UUID-Only Rendering The HF renderer first attempts to render a conversation without loading media when -all its media items are image/video URLs with UUIDs and at least one URL is supplied. -This is enabled by default. Set `skip_early_mm_lookup=True` to opt out per request. +every media item has a UUID and at least one image, audio, or video item is supplied as +a URL or inline data. This is enabled by default. Set `skip_early_mm_lookup=True` to +opt out per request. - If every media item is in the processor cache, loading and decoding are skipped. -- If any item misses, vLLM retries that conversation once with all the original URLs, - loading the supplied media normally. Other conversations in the batch are not retried. +- If any item misses, vLLM retries that conversation once, loading the supplied media + normally. Other conversations in the batch are not retried. This optimization requires the processor cache and is disabled for unified vision -chunks, audio extracted from video, and conversations containing other media types. -An item supplied without a URL must still be cached for the request to succeed. +chunks and audio extracted from video. An item supplied without a URL must still be +cached for the request to succeed. The UUID is authoritative on a cache hit; vLLM does not verify that the URL still identifies the cached media (see [Multimodal Media UUID Security](../usage/security.md#multimodal-media-uuid-security)). diff --git a/tests/renderers/chat_utils/test_audio.py b/tests/renderers/chat_utils/test_audio.py index bbfe4ca8de81..392d4355a055 100644 --- a/tests/renderers/chat_utils/test_audio.py +++ b/tests/renderers/chat_utils/test_audio.py @@ -1,8 +1,12 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio +from unittest.mock import AsyncMock, Mock + import pytest from vllm.config import ModelConfig +from vllm.multimodal.media import MediaConnector from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async from vllm.utils.serial_utils import tensor2base64 @@ -222,3 +226,43 @@ async def test_parse_chat_messages_audio_embeds_async( assert mm_uuids is not None assert "audio" in mm_uuids assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None]) + + +@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize( + "part", + [ + {"type": "audio_url", "audio_url": {"url": "https://example.com/a.wav"}}, + {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}, + ], + ids=["audio_url", "input_audio"], +) +def test_parse_chat_messages_defers_audio_with_uuid( + qwen2_audio_model_config, monkeypatch, use_async, part +): + """Audio with a UUID is not loaded when deferring to a cache lookup.""" + fetch_audio = AsyncMock() if use_async else Mock() + method = "fetch_audio_async" if use_async else "fetch_audio" + monkeypatch.setattr(MediaConnector, method, fetch_audio) + messages = [ + { + "role": "user", + "content": [ + {**part, "uuid": "abcd"}, + {"type": "text", "text": "What does the audio say?"}, + ], + } + ] + + parse = parse_chat_messages_async if use_async else parse_chat_messages + result = parse( + messages, + qwen2_audio_model_config, + content_format="string", + defer_uuid_media=True, + ) + _, mm_data, mm_uuids = asyncio.run(result) if use_async else result + + fetch_audio.assert_not_called() + assert mm_data == {"audio": [None]} + assert mm_uuids == {"audio": ["abcd"]} diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index c3a6cc80a148..5d8fb98a52d4 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -7,7 +7,6 @@ from unittest.mock import AsyncMock, Mock import pytest -from PIL import Image from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset @@ -16,7 +15,7 @@ from vllm.multimodal.media import MediaConnector from vllm.multimodal.parse import parse_mm_uuids from vllm.multimodal.processing.processor import MultiModalProcessorCacheMissError -from vllm.renderers.base import BaseRenderer +from vllm.renderers.chat_utils import can_conversation_defer_media from vllm.renderers.hf import HfRenderer from vllm.renderers.params import ChatParams from vllm.tokenizers.registry import cached_tokenizer_from_config @@ -28,34 +27,15 @@ baby_reading_video_asset.np_ndarrays, baby_reading_video_asset.metadata, ) -kimi_vision_chunk_image = { - "type": "image", - "image": cherry_pil_image, - "uuid": "test-image-uuid", -} -kimi_vision_chunk_video = { - "type": "video_chunk", - "video_chunk": [ - Image.fromarray(frame) for frame in baby_reading_video_asset.np_ndarrays[:4] - ], - "uuid": "test-video-uuid", - "video_idx": 0, - "prompt": ("<|media_begin|>video<|media_content|><|media_pad|><|media_end|>"), -} def _build_renderer( - *, - model: str = "Qwen/Qwen2.5-VL-3B-Instruct", - mm_cache_gb: float = 4.0, - enable_prefix_caching: bool = True, - trust_remote_code: bool = False, + *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True ) -> HfRenderer: model_config = ModelConfig( - model=model, + model="Qwen/Qwen2.5-VL-3B-Instruct", max_model_len=128, mm_processor_cache_gb=mm_cache_gb, - trust_remote_code=trust_remote_code, ) vllm_config = VllmConfig( @@ -149,23 +129,8 @@ def test_cached_uuid_skips_url_loading( use_async: bool, ): renderer = _build_renderer() - media_url = f"https://example.com/test.{modality}" - media_uuid = f"test-{modality}-uuid" fetch_media = _mock_media(monkeypatch, modality, media, use_async) - - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": f"Describe this {modality}."}, - { - "type": f"{modality}_url", - f"{modality}_url": {"url": media_url}, - "uuid": media_uuid, - }, - ], - } - ] + messages = _media_messages("Describe", (modality, f"test-{modality}-uuid")) params = ChatParams(skip_early_mm_lookup=skip_early_mm_lookup) _, first_prompts = _render_chat(renderer, [messages], params, use_async) @@ -192,26 +157,8 @@ def test_uuid_cache_eviction_falls_back_to_url( media: object, ): renderer = _build_renderer() - mm_processor_cache = renderer.mm_processor_cache - assert mm_processor_cache is not None - - fetch_media = Mock(return_value=media) - monkeypatch.setattr(MediaConnector, f"fetch_{modality}", fetch_media) - - media_url = f"https://example.com/test.{modality}" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": f"Describe this {modality}."}, - { - "type": f"{modality}_url", - f"{modality}_url": {"url": media_url}, - "uuid": f"test-{modality}-uuid", - }, - ], - } - ] + fetch_media = _mock_media(monkeypatch, modality, media, use_async=False) + messages = _media_messages("Describe", (modality, f"test-{modality}-uuid")) renderer.render_chat([messages], ChatParams()) renderer.clear_mm_cache() @@ -249,8 +196,7 @@ def test_partial_uuid_hit_retries_only_affected_conversation(monkeypatch, use_as assert [ call.args[0][0]["content"][0]["text"] for call in render_messages.call_args_list - ].count("Cached conversation") == 1 - assert render_messages.call_count == 3 + ] == ["Partial conversation"] fetch_image.assert_called_once_with("https://example.com/partial-image.image") assert fetch_video.call_count == 1 assert prompts[0]["prompt_token_ids"] == warm_prompts[0]["prompt_token_ids"] @@ -258,173 +204,151 @@ def test_partial_uuid_hit_retries_only_affected_conversation(monkeypatch, use_as assert set(prompts[1]["mm_hashes"]) == {"image", "video"} assert conversations == original - fetch_image.reset_mock() - fetch_video.reset_mock() - _, cached_prompts = _render_chat(renderer, conversations, params, use_async) - fetch_image.assert_not_called() - fetch_video.assert_not_called() - assert [p["prompt_token_ids"] for p in cached_prompts] == [ - p["prompt_token_ids"] for p in prompts - ] - - -def test_async_uuid_render_preserves_cache_update_order(monkeypatch): - """Sender cache updates must follow the order of returned conversations.""" - renderer = _build_renderer() - conversations = [ - _media_messages("first", ("image", "shared")), - _media_messages("second", ("image", "shared")), - ] - completed = [] - - async def render(_self, batch, **kwargs): - name = batch[0][0]["content"][0]["text"] - if name == "first": - await asyncio.sleep(0) - completed.append(name) - return [[]], [{"prompt": name}] - - monkeypatch.setattr(BaseRenderer, "render_chat_async", render) - _, prompts = _render_chat(renderer, conversations, ChatParams(), True) - - assert completed == [prompt["prompt"] for prompt in prompts] == ["first", "second"] +@pytest.mark.parametrize( + ("content", "expected"), + [ + pytest.param( + [{"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"}], + True, + id="image-url", + ), + pytest.param( + [ + { + "type": "input_audio", + "input_audio": {"data": "AAAA", "format": "wav"}, + "uuid": "audio", + } + ], + True, + id="input-audio", + ), + pytest.param( + [{"type": "image_url", "image_url": None, "uuid": "image"}], + False, + id="uuid-without-data", + ), + pytest.param( + [ + { + "type": "input_audio", + "input_audio": {"data": "", "format": "wav"}, + "uuid": "audio", + } + ], + False, + id="input-audio-without-data", + ), + pytest.param( + [ + {"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"}, + {"type": "video_url", "video_url": {"url": "url"}}, + ], + False, + id="missing-uuid", + ), + pytest.param("Describe", False, id="text-only"), + ], +) +def test_can_conversation_defer_media(content, expected): + """Media loading is deferred only if all media has UUIDs and some has data.""" + messages = [{"role": "user", "content": content}] -@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) -def test_missing_uuid_loads_all_media(monkeypatch, use_async): - renderer = _build_renderer() - fetch_image = _mock_media(monkeypatch, "image", cherry_pil_image, use_async) - messages = _media_messages("Describe", ("image", "cached-image")) - _render_chat(renderer, [messages], ChatParams(), use_async) - messages[0]["content"].append( - { - "type": "image_url", - "image_url": {"url": "https://example.com/no-uuid.image"}, - } - ) - fetch_image.reset_mock() - - _render_chat(renderer, [messages], ChatParams(), use_async) + assert can_conversation_defer_media(messages) is expected - assert fetch_image.call_count == 2 +def test_can_conversation_defer_media_does_not_consume_iterators(): + part = {"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"} + content = iter([part]) -@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) -def test_uuid_without_url_still_requires_cache_hit(monkeypatch, use_async): - renderer = _build_renderer() - fetch_image = _mock_media(monkeypatch, "image", cherry_pil_image, use_async) - messages = _media_messages("Describe", ("image", "missing-image")) - messages[0]["content"][1]["image_url"]["url"] = None + assert not can_conversation_defer_media([{"role": "user", "content": content}]) + assert next(content) == part - with pytest.raises(MultiModalProcessorCacheMissError, match="data is not provided"): - _render_chat(renderer, [messages], ChatParams(), use_async) - fetch_image.assert_not_called() +def _wrap_process_for_engine(monkeypatch, renderer, use_async): + method = "process_for_engine_async" if use_async else "process_for_engine" + mock_type = AsyncMock if use_async else Mock + process = mock_type(wraps=getattr(renderer, method)) + monkeypatch.setattr(renderer, method, process) + return process @pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize( - "case", ["skip-cache", "no-cache", "audio-in-video", "other-media"] + "case", ["skip-cache", "no-cache", "audio-in-video", "missing-uuid"] ) -def test_ineligible_conversation_renders_original_messages( - monkeypatch, use_async, case -): +def test_ineligible_conversation_loads_media_upfront(monkeypatch, use_async, case): + """Conversations that cannot defer media are processed once, not retried.""" renderer = _build_renderer(mm_cache_gb=0 if case == "no-cache" else 4) + fetch_image = _mock_media(monkeypatch, "image", cherry_pil_image, use_async) messages = _media_messages("Describe", ("image", "image")) + if case == "missing-uuid": + del messages[0]["content"][1]["uuid"] params = ChatParams() if case == "audio-in-video": params = ChatParams(mm_processor_kwargs={"use_audio_in_video": True}) - elif case == "other-media": - messages[0]["content"].append( - { - "type": "audio_url", - "audio_url": {"url": "https://example.com/audio.wav"}, - "uuid": "audio", - } - ) - mock_type = AsyncMock if use_async else Mock - render = mock_type(return_value=([[]], [{}])) - method = "render_chat_async" if use_async else "render_chat" - monkeypatch.setattr(BaseRenderer, method, render) + process = _wrap_process_for_engine(monkeypatch, renderer, use_async) _render_chat( - renderer, [messages], params, use_async, skip_mm_cache=case == "skip-cache" + renderer, + [messages], + params, + use_async, + skip_mm_cache=case == "skip-cache", ) - assert render.call_count == 1 - assert render.call_args.args[0][0] is messages + assert process.call_count == 1 + assert fetch_image.call_count == 1 @pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) -@pytest.mark.parametrize( - "error", [ValueError("invalid media"), MultiModalProcessorCacheMissError("missing")] -) -def test_uuid_render_retry_is_bounded_and_only_handles_cache_misses( - monkeypatch, use_async, error -): +def test_uuid_only_cache_miss_is_not_retried(monkeypatch, use_async): + """Without a URL to fall back to, a cache miss fails without a retry.""" renderer = _build_renderer() messages = _media_messages("Describe", ("image", "image")) - mock_type = AsyncMock if use_async else Mock - render = mock_type(side_effect=error) - method = "render_chat_async" if use_async else "render_chat" - monkeypatch.setattr(BaseRenderer, method, render) + messages[0]["content"][1]["image_url"] = None + process = _wrap_process_for_engine(monkeypatch, renderer, use_async) - with pytest.raises(type(error), match=str(error)): + with pytest.raises(MultiModalProcessorCacheMissError): _render_chat(renderer, [messages], ChatParams(), use_async) - assert render.call_count == ( - 2 if isinstance(error, MultiModalProcessorCacheMissError) else 1 - ) + assert process.call_count == 1 -@pytest.mark.parametrize( - ("modality", "media"), - [ - pytest.param("image", kimi_vision_chunk_image, id="image"), - pytest.param("video", kimi_vision_chunk_video, id="video"), - ], -) -def test_early_uuid_lookup_is_disabled_for_unified_vision_chunks( - monkeypatch: pytest.MonkeyPatch, - modality: str, - media: object, -): - renderer = _build_renderer( - model="moonshotai/Kimi-K2.5", - trust_remote_code=True, - ) - mm_processor_cache = renderer.mm_processor_cache - assert mm_processor_cache is not None - - is_cached_item = Mock(return_value=True) - monkeypatch.setattr(mm_processor_cache, "is_cached_item", is_cached_item) - fetch_media = Mock(return_value=media) - monkeypatch.setattr(MediaConnector, f"fetch_{modality}", fetch_media) - - media_url = f"https://example.com/test.{modality}" - messages = [ - { - "role": "user", - "content": [ - {"type": "text", "text": f"Describe this {modality}."}, - { - "type": f"{modality}_url", - f"{modality}_url": {"url": media_url}, - "uuid": f"test-{modality}-uuid", - }, - ], - } +def test_batch_cache_misses_reload_media_concurrently(monkeypatch): + """Retried conversations load media concurrently and keep the batch order.""" + renderer = _build_renderer() + active_fetches = max_active_fetches = 0 + + async def fetch_image(*args, **kwargs): + nonlocal active_fetches, max_active_fetches + active_fetches += 1 + max_active_fetches = max(max_active_fetches, active_fetches) + try: + await asyncio.sleep(0.01) + return cherry_pil_image + finally: + active_fetches -= 1 + + monkeypatch.setattr(MediaConnector, "fetch_image_async", fetch_image) + conversations = [ + _media_messages(f"Conversation {i}", ("image", f"image-{i}")) for i in range(2) ] - monkeypatch.setattr( - renderer, "process_for_engine", lambda prompt, *args, **kwargs: prompt + out_conversations, prompts = _render_chat( + renderer, conversations, ChatParams(), use_async=True + ) + _, cached_prompts = _render_chat( + renderer, conversations, ChatParams(), use_async=True ) - conversations, prompts = renderer.render_chat([messages], ChatParams()) - assert len(conversations[0]) == 1 - assert prompts[0]["multi_modal_data"] == {"vision_chunk": [media]} - assert fetch_media.call_count == 1 - is_cached_item.assert_not_called() + assert max_active_fetches == 2 + assert all( + f"Conversation {i}" in str(conversation) + for i, conversation in enumerate(out_conversations) + ) + assert [p["mm_hashes"] for p in prompts] == [p["mm_hashes"] for p in cached_prompts] def test_multi_modal_uuids_length_mismatch_raises(): diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index f22b4215bdbd..6934c586fc28 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -9,7 +9,7 @@ from contextlib import ExitStack from dataclasses import replace from functools import cached_property -from typing import TYPE_CHECKING, Any, Generic, overload +from typing import TYPE_CHECKING, Any, Generic, cast, overload from typing_extensions import TypeVar @@ -45,6 +45,7 @@ ) from vllm.multimodal.processing import BaseMultiModalProcessor from vllm.multimodal.processing import ProcessorInputs as MMProcessorInputs +from vllm.multimodal.processing.processor import MultiModalProcessorCacheMissError from vllm.multimodal.registry import MultiModalTimingRegistry from vllm.tokenizers import TokenizerLike from vllm.utils.async_utils import make_async @@ -52,6 +53,7 @@ from vllm.utils.torch_utils import set_default_torch_num_threads from vllm.v1.metrics.stats import MultiModalCacheStats +from .chat_utils import can_conversation_defer_media, uses_audio_in_video from .embed_utils import safe_load_prompt_embeds from .inputs import ( DictPrompt, @@ -1292,6 +1294,106 @@ async def render_cmpl_async( ) ) + def _can_defer_uuid_media(self) -> bool: + """Whether this renderer implements `_render_messages_deferring_media`.""" + return False + + def _render_messages_deferring_media( + self, + messages: list["ChatCompletionMessageParam"], + params: ChatParams, + ) -> tuple[list["ConversationMessage"], DictPrompt]: + """Like `render_messages`, but without loading media that has a UUID.""" + raise NotImplementedError + + async def _render_messages_deferring_media_async( + self, + messages: list["ChatCompletionMessageParam"], + params: ChatParams, + ) -> tuple[list["ConversationMessage"], DictPrompt]: + return self._render_messages_deferring_media(messages, params) + + def _should_defer_uuid_media( + self, chat_params: ChatParams, skip_mm_cache: bool + ) -> bool: + return ( + self._can_defer_uuid_media() + and self._mm_processor_cache is not None + and not skip_mm_cache + and not chat_params.skip_early_mm_lookup + and not uses_audio_in_video(chat_params.mm_processor_kwargs) + ) + + def _render_chat_prompts( + self, + conversations: Sequence[list["ChatCompletionMessageParam"]], + chat_params: ChatParams, + tok_params: TokenizeParams, + prompt_extras: dict[str, Any], + *, + defer_uuid_media: bool, + ) -> tuple[list[list["ConversationMessage"]], list[TokPrompt], list[bool]]: + """Render and tokenize `conversations`. + + Returns: + The rendered conversations, their prompts, and whether each prompt + had its media loading deferred. + + """ + + def render(conversation: list["ChatCompletionMessageParam"]): + deferred = defer_uuid_media and can_conversation_defer_media(conversation) + render_messages = ( + self._render_messages_deferring_media + if deferred + else self.render_messages + ) + return *render_messages(conversation, chat_params), deferred + + rendered = [render(conversation) for conversation in conversations] + + tok_prompts = self.tokenize_prompts( + [prompt for _, prompt, _ in rendered], tok_params + ) + self._apply_prompt_extras(tok_prompts, prompt_extras) + + return ( + [conv for conv, _, _ in rendered], + tok_prompts, + [deferred for _, _, deferred in rendered], + ) + + async def _render_chat_prompts_async( + self, + conversations: Sequence[list["ChatCompletionMessageParam"]], + chat_params: ChatParams, + tok_params: TokenizeParams, + prompt_extras: dict[str, Any], + *, + defer_uuid_media: bool, + ) -> tuple[list[list["ConversationMessage"]], list[TokPrompt], list[bool]]: + async def render(conversation: list["ChatCompletionMessageParam"]): + deferred = defer_uuid_media and can_conversation_defer_media(conversation) + render_messages = ( + self._render_messages_deferring_media_async + if deferred + else self.render_messages_async + ) + return *await render_messages(conversation, chat_params), deferred + + rendered = await asyncio.gather(*(render(c) for c in conversations)) + + tok_prompts = await self.tokenize_prompts_async( + [prompt for _, prompt, _ in rendered], tok_params + ) + self._apply_prompt_extras(tok_prompts, prompt_extras) + + return ( + [conv for conv, _, _ in rendered], + tok_prompts, + [deferred for _, _, deferred in rendered], + ) + def render_chat( self, conversations: Sequence[list["ChatCompletionMessageParam"]], @@ -1306,31 +1408,46 @@ def render_chat( if tok_params is None: tok_params = self.default_chat_tok_params - rendered = [ - self.render_messages(conversation, chat_params) - for conversation in conversations - ] - - out_conversations = list[list["ConversationMessage"]]() - dict_prompts = list[DictPrompt]() - for conv, prompt in rendered: - out_conversations.append(conv) - dict_prompts.append(prompt) - - tok_prompts = self.tokenize_prompts(dict_prompts, tok_params) - prompt_extras = dict(prompt_extras or {}) prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {} - self._apply_prompt_extras(tok_prompts, prompt_extras) - eng_prompts = [ - self.process_for_engine( - prompt, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params + out_conversations, tok_prompts, deferred = self._render_chat_prompts( + conversations, + chat_params, + tok_params, + prompt_extras, + defer_uuid_media=self._should_defer_uuid_media(chat_params, skip_mm_cache), + ) + + def process(prompt: TokPrompt, is_deferred: bool = False): + try: + return self.process_for_engine( + prompt, + arrival_time, + skip_mm_cache=skip_mm_cache, + tok_params=tok_params, + ) + except MultiModalProcessorCacheMissError: + if not is_deferred: + raise + return None + + eng_prompts = [process(p, d) for p, d in zip(tok_prompts, deferred)] + + # Retry the deferred prompts that missed the cache with their media loaded. + if missed := [i for i, prompt in enumerate(eng_prompts) if prompt is None]: + retry_conversations, retry_prompts, _ = self._render_chat_prompts( + [conversations[i] for i in missed], + chat_params, + tok_params, + prompt_extras, + defer_uuid_media=False, ) - for prompt in tok_prompts - ] + for i, conv, prompt in zip(missed, retry_conversations, retry_prompts): + out_conversations[i] = conv + eng_prompts[i] = process(prompt) - return out_conversations, eng_prompts + return out_conversations, cast(list[EngineInput], eng_prompts) async def render_chat_async( self, @@ -1346,30 +1463,52 @@ async def render_chat_async( if tok_params is None: tok_params = self.default_chat_tok_params - rendered = [ - self.render_messages_async(conversation, chat_params) - for conversation in conversations - ] - - out_conversations = list[list["ConversationMessage"]]() - dict_prompts = list[DictPrompt]() - for conv, prompt in await asyncio.gather(*rendered): - out_conversations.append(conv) - dict_prompts.append(prompt) - - tok_prompts = await self.tokenize_prompts_async(dict_prompts, tok_params) - prompt_extras = dict(prompt_extras or {}) prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {} - self._apply_prompt_extras(tok_prompts, prompt_extras) - eng_prompts = await asyncio.gather( - *( - self.process_for_engine_async( - p, arrival_time, skip_mm_cache=skip_mm_cache, tok_params=tok_params + rendered = await self._render_chat_prompts_async( + conversations, + chat_params, + tok_params, + prompt_extras, + defer_uuid_media=self._should_defer_uuid_media(chat_params, skip_mm_cache), + ) + out_conversations, tok_prompts, deferred = rendered + + async def process(prompt: TokPrompt, is_deferred: bool = False): + try: + return await self.process_for_engine_async( + prompt, + arrival_time, + skip_mm_cache=skip_mm_cache, + tok_params=tok_params, ) - for p in tok_prompts - ) + except MultiModalProcessorCacheMissError: + if not is_deferred: + raise + return None + + eng_prompts = await asyncio.gather( + *(process(p, d) for p, d in zip(tok_prompts, deferred)) ) - return out_conversations, eng_prompts + # Retry the deferred prompts that missed the cache with their media loaded. + if missed := [i for i, prompt in enumerate(eng_prompts) if prompt is None]: + retried = await self._render_chat_prompts_async( + [conversations[i] for i in missed], + chat_params, + tok_params, + prompt_extras, + defer_uuid_media=False, + ) + retry_conversations, retry_prompts, _ = retried + retry_eng_prompts = await asyncio.gather( + *(process(prompt) for prompt in retry_prompts) + ) + for i, conv, eng_prompt in zip( + missed, retry_conversations, retry_eng_prompts + ): + out_conversations[i] = conv + eng_prompts[i] = eng_prompt + + return out_conversations, cast(list[EngineInput], eng_prompts) diff --git a/vllm/renderers/chat_utils/__init__.py b/vllm/renderers/chat_utils/__init__.py index dd68b7c620c3..e418ef963c06 100644 --- a/vllm/renderers/chat_utils/__init__.py +++ b/vllm/renderers/chat_utils/__init__.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from .chat_message import ( + can_conversation_defer_media, has_non_text_content, parse_chat_message_content, parse_chat_message_content_parts, @@ -13,7 +14,7 @@ load_chat_template, validate_chat_template, ) -from .content_parser import PROMPT_EMBEDS_PLACEHOLDER_TOKEN +from .content_parser import PROMPT_EMBEDS_PLACEHOLDER_TOKEN, uses_audio_in_video from .item_tracker import ( AsyncMultiModalItemTracker, BaseMultiModalItemTracker, @@ -33,10 +34,12 @@ "BaseMultiModalItemTracker", "MultiModalItemTracker", "AsyncMultiModalItemTracker", + "can_conversation_defer_media", "has_non_text_content", "parse_chat_message_content", "parse_chat_message_content_parts", "parse_chat_messages", "parse_chat_messages_async", "PROMPT_EMBEDS_PLACEHOLDER_TOKEN", + "uses_audio_in_video", ] diff --git a/vllm/renderers/chat_utils/chat_message.py b/vllm/renderers/chat_utils/chat_message.py index bc4ae23f8337..e6122f1b62aa 100644 --- a/vllm/renderers/chat_utils/chat_message.py +++ b/vllm/renderers/chat_utils/chat_message.py @@ -345,6 +345,18 @@ def parse_chat_message_content_mm_part( _TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} # Keys that mark a content part as multimodal, whatever its ``type``. _MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES +# Content part types whose media is loaded from a URL or inline data. +_LOADED_MEDIA_PART_TYPES = frozenset( + {"image_url", "audio_url", "input_audio", "video_url"} +) + + +def _is_non_text_part(part: object) -> bool: + return isinstance(part, dict) and ( + any(key in part for key in _MEDIA_CONTENT_PART_KEYS) + or not isinstance(part_type := part.get("type", "text"), str) + or part_type not in _TEXT_CONTENT_PART_TYPES + ) def has_non_text_content(messages: Any) -> bool: @@ -359,16 +371,37 @@ def has_non_text_content(messages: Any) -> bool: content = msg.get("content") if isinstance(msg, dict) else None if not isinstance(content, list): continue - for part in content: - if isinstance(part, dict) and ( - any(key in part for key in _MEDIA_CONTENT_PART_KEYS) - or not isinstance(part_type := part.get("type", "text"), str) - or part_type not in _TEXT_CONTENT_PART_TYPES - ): - return True + if any(_is_non_text_part(part) for part in content): + return True return False +def can_conversation_defer_media(messages: list[ChatCompletionMessageParam]) -> bool: + """Whether every media part in ``messages`` has a UUID and at least one + has data to load. + + Only list content is inspected, so content that is a one-shot iterator is + never consumed; messages with such content cannot defer. + """ + has_data = False + for message in messages: + content = message.get("content") + if content is None or isinstance(content, str): + continue + if not isinstance(content, list): + return False + for part in content: + if not _is_non_text_part(part): + continue + if part.get("uuid") is None: + return False + part_type, data = parse_chat_message_content_mm_part(part) + if part_type in _LOADED_MEDIA_PART_TYPES: + # `input_audio` carries its data inside a dict. + has_data |= bool(data.get("data") if isinstance(data, dict) else data) + return has_data + + def parse_chat_message_content_parts( role: str, parts: Iterable[ChatCompletionContentPartParam], @@ -752,6 +785,7 @@ def parse_chat_messages( content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, + defer_uuid_media: bool = False, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -761,6 +795,7 @@ def parse_chat_messages( mm_tracker = MultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, + defer_uuid_media=defer_uuid_media, ) for msg in messages: @@ -791,6 +826,7 @@ async def parse_chat_messages_async( content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, + defer_uuid_media: bool = False, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -800,6 +836,7 @@ async def parse_chat_messages_async( mm_tracker = AsyncMultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, + defer_uuid_media=defer_uuid_media, ) for msg in messages: diff --git a/vllm/renderers/chat_utils/content_parser.py b/vllm/renderers/chat_utils/content_parser.py index 72c6f0d00629..bddc6f6905f7 100644 --- a/vllm/renderers/chat_utils/content_parser.py +++ b/vllm/renderers/chat_utils/content_parser.py @@ -46,7 +46,16 @@ ) +def uses_audio_in_video(mm_processor_kwargs: dict[str, Any] | None) -> bool: + """Whether audio is extracted from video content parts.""" + return bool( + mm_processor_kwargs and mm_processor_kwargs.get("use_audio_in_video", False) + ) + + class BaseMultiModalContentParser(ABC): + _mm_processor_kwargs: dict[str, Any] | None + def __init__(self) -> None: super().__init__() @@ -72,6 +81,10 @@ def _add_placeholder(self, modality: ModalityStr, placeholder: str | None): def mm_placeholder_storage(self) -> dict[str, list]: return dict(self._placeholder_storage) + @cached_property + def _use_audio_in_video(self) -> bool: + return uses_audio_in_video(self._mm_processor_kwargs) + @abstractmethod def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: raise NotImplementedError @@ -171,6 +184,7 @@ def parse_prompt_embeds(self, data: str) -> None: self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN) def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: + image_url = self._tracker.defer_media_url(image_url, uuid) image = self._connector.fetch_image(image_url) if image_url else None placeholder = self._tracker.add("image", (image, uuid)) @@ -241,6 +255,7 @@ def parse_image_pil( self._add_placeholder("image", placeholder) def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None: + audio_url = self._tracker.defer_media_url(audio_url, uuid) audio = self._connector.fetch_audio(audio_url) if audio_url else None placeholder = self._tracker.add("audio", (audio, uuid)) @@ -263,6 +278,7 @@ def parse_input_audio( return self.parse_audio(audio_url, uuid) def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: + video_url = self._tracker.defer_media_url(video_url, uuid) video = ( self._connector.fetch_video( video_url=video_url, @@ -276,11 +292,7 @@ def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: self._add_placeholder("video", placeholder) # Extract audio from video if use_audio_in_video is True - if ( - video_url - and self._mm_processor_kwargs - and self._mm_processor_kwargs.get("use_audio_in_video", False) - ): + if video_url and self._use_audio_in_video: audio = self._connector.fetch_audio(video_url) if video_url else None audio_placeholder = self._tracker.add("audio", (audio, uuid)) self._add_placeholder("audio", audio_placeholder) @@ -376,6 +388,7 @@ async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None): return image, uuid def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: + image_url = self._tracker.defer_media_url(image_url, uuid) placeholder = self._tracker.add( "image", partial(self._image_with_uuid_async, image_url, uuid) ) @@ -468,6 +481,7 @@ async def _audio_with_uuid_async(self, audio_url: str | None, uuid: str | None): return audio, uuid def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None: + audio_url = self._tracker.defer_media_url(audio_url, uuid) placeholder = self._tracker.add( "audio", partial(self._audio_with_uuid_async, audio_url, uuid) ) @@ -501,17 +515,14 @@ async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None): return video, uuid def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: + video_url = self._tracker.defer_media_url(video_url, uuid) placeholder = self._tracker.add( "video", partial(self._video_with_uuid_async, video_url, uuid) ) self._add_placeholder("video", placeholder) # Extract audio from video if use_audio_in_video is True - if ( - video_url - and self._mm_processor_kwargs - and self._mm_processor_kwargs.get("use_audio_in_video", False) - ): + if video_url and self._use_audio_in_video: audio_placeholder = self._tracker.add( "audio", partial(self._audio_with_uuid_async, video_url, uuid) ) diff --git a/vllm/renderers/chat_utils/item_tracker.py b/vllm/renderers/chat_utils/item_tracker.py index 4791ae1f1fdc..11e7f8502142 100644 --- a/vllm/renderers/chat_utils/item_tracker.py +++ b/vllm/renderers/chat_utils/item_tracker.py @@ -207,11 +207,13 @@ def __init__( self, model_config: ModelConfig, media_io_kwargs: dict[str, dict[str, Any]] | None = None, + defer_uuid_media: bool = False, ): super().__init__() self._model_config = model_config self._media_io_kwargs = media_io_kwargs + self._defer_uuid_media = defer_uuid_media self._items_by_modality = defaultdict[str, list[_T]](list) # Track original modality for each vision_chunk item (image or video) @@ -263,6 +265,10 @@ def mm_processor(self): def video_processor_name(self) -> str | None: return get_video_processor_cls_name(self.model_config) + def defer_media_url(self, url: str | None, uuid: str | None) -> str | None: + """Return `None` in place of `url` if loading media with a UUID is deferred.""" + return None if self._defer_uuid_media and uuid is not None else url + def add(self, modality: ModalityStr, item: _T) -> str | None: """Add a multi-modal item to the current prompt and returns the placeholder string to use, if any. diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 76e35686f46b..68deb5875c4f 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -9,7 +9,7 @@ import weakref from collections import defaultdict, deque from collections.abc import Sequence -from functools import lru_cache, partial +from functools import lru_cache from typing import TYPE_CHECKING, Any, Final, Literal, cast, overload import jinja2 @@ -35,7 +35,6 @@ PlaceholderRange, ) from vllm.multimodal.processing.processor import ( - MultiModalProcessorCacheMissError, PromptReplacement, apply_token_matches, find_mm_placeholders, @@ -73,7 +72,7 @@ ) from .inputs import DictPrompt - from .params import ChatParams, TokenizeParams + from .params import ChatParams logger = init_logger(__name__) @@ -1013,179 +1012,31 @@ def _can_produce_offsets(self) -> bool: # Only fast tokenizers expose offset_mapping. return self.tokenizer is not None and self.tokenizer.is_fast - def _get_uuid_only_messages( + @override + def _can_defer_uuid_media(self) -> bool: + return not self.use_unified_vision_chunk + + def render_messages( self, messages: list[ChatCompletionMessageParam], params: ChatParams, - ) -> list[ChatCompletionMessageParam] | None: - """Omit URLs only when every media item can be looked up by UUID.""" - if ( - params.skip_early_mm_lookup - or self.mm_processor_cache is None - or self.use_unified_vision_chunk - ): - return None - - mm_config = self.model_config.get_multimodal_config() - processor_kwargs = mm_config.merge_mm_processor_kwargs( - params.mm_processor_kwargs or {} - ) - if processor_kwargs.get("use_audio_in_video", False): - return None - - uuid_only_messages = [] - has_url = False - for message in messages: - content = message.get("content") - if content is None or isinstance(content, str): - uuid_only_messages.append(message) - continue - if not isinstance(content, list): - return None - - uuid_only_content: list[Any] = [] - for part in content: - if isinstance(part, str) or part.get("type") in ( - "text", - "input_text", - "output_text", - "refusal", - "thinking", - "tool_reference", - ): - uuid_only_content.append(part) - continue - if part.get("type") not in ( - None, - "image_url", - "input_image", - "video_url", - ): - return None - url_key = next( - (key for key in ("image_url", "video_url") if key in part), None - ) - if url_key is None or part.get("uuid") is None: - return None - - media = cast(dict[str, Any], part)[url_key] - url = media.get("url") if isinstance(media, dict) else media - has_url |= url is not None - uuid_only_content.append( - { - **part, - url_key: {**media, "url": None} - if isinstance(media, dict) - else None, - } - ) - - uuid_only_messages.append( - cast( - "ChatCompletionMessageParam", - {**message, "content": uuid_only_content}, - ) - ) - - return uuid_only_messages if has_url else None - - @override - def render_chat( - self, - conversations: Sequence[list[ChatCompletionMessageParam]], - chat_params: ChatParams, - tok_params: TokenizeParams | None = None, - *, - prompt_extras: dict[str, Any] | None = None, - skip_mm_cache: bool = False, - ): - render = partial( - super().render_chat, - chat_params=chat_params, - tok_params=tok_params, - prompt_extras=prompt_extras, - skip_mm_cache=skip_mm_cache, - ) - - uuid_only_conversations = [ - None - if skip_mm_cache - else self._get_uuid_only_messages(messages, chat_params) - for messages in conversations - ] - if all(messages is None for messages in uuid_only_conversations): - return render(conversations) - - def render_one( - messages: list[ChatCompletionMessageParam], - uuid_only: list[ChatCompletionMessageParam] | None, - ): - if uuid_only is not None: - try: - return render([uuid_only]) - except MultiModalProcessorCacheMissError: - pass - return render([messages]) - - rendered = [ - render_one(messages, uuid_only) - for messages, uuid_only in zip(conversations, uuid_only_conversations) - ] - return [result[0][0] for result in rendered], [ - result[1][0] for result in rendered - ] + ) -> tuple[list[ConversationMessage], DictPrompt]: + return self._render_messages(messages, params, defer_uuid_media=False) @override - async def render_chat_async( + def _render_messages_deferring_media( self, - conversations: Sequence[list[ChatCompletionMessageParam]], - chat_params: ChatParams, - tok_params: TokenizeParams | None = None, - *, - prompt_extras: dict[str, Any] | None = None, - skip_mm_cache: bool = False, - ): - render = partial( - super().render_chat_async, - chat_params=chat_params, - tok_params=tok_params, - prompt_extras=prompt_extras, - skip_mm_cache=skip_mm_cache, - ) - - uuid_only_conversations = [ - None - if skip_mm_cache - else self._get_uuid_only_messages(messages, chat_params) - for messages in conversations - ] - if all(messages is None for messages in uuid_only_conversations): - return await render(conversations) - - async def render_one( - messages: list[ChatCompletionMessageParam], - uuid_only: list[ChatCompletionMessageParam] | None, - ): - if uuid_only is not None: - try: - return await render([uuid_only]) - except MultiModalProcessorCacheMissError: - pass - return await render([messages]) - - # Sender-cache updates must follow the order of returned conversations. - rendered = [ - await render_one(messages, uuid_only) - for messages, uuid_only in zip(conversations, uuid_only_conversations) - ] - return [result[0][0] for result in rendered], [ - result[1][0] for result in rendered - ] + messages: list[ChatCompletionMessageParam], + params: ChatParams, + ) -> tuple[list[ConversationMessage], DictPrompt]: + return self._render_messages(messages, params, defer_uuid_media=True) - def render_messages( + def _render_messages( self, messages: list[ChatCompletionMessageParam], params: ChatParams, + *, + defer_uuid_media: bool, ) -> tuple[list[ConversationMessage], DictPrompt]: model_config = self.model_config tokenizer = self.get_tokenizer() @@ -1208,6 +1059,7 @@ def render_messages( ), media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, + defer_uuid_media=defer_uuid_media, ) # prompt_embeds tensors are carried by the tracker through mm_data, @@ -1293,6 +1145,27 @@ async def render_messages_async( self, messages: list[ChatCompletionMessageParam], params: ChatParams, + ) -> tuple[list[ConversationMessage], DictPrompt]: + return await self._render_messages_async( + messages, params, defer_uuid_media=False + ) + + @override + async def _render_messages_deferring_media_async( + self, + messages: list[ChatCompletionMessageParam], + params: ChatParams, + ) -> tuple[list[ConversationMessage], DictPrompt]: + return await self._render_messages_async( + messages, params, defer_uuid_media=True + ) + + async def _render_messages_async( + self, + messages: list[ChatCompletionMessageParam], + params: ChatParams, + *, + defer_uuid_media: bool, ) -> tuple[list[ConversationMessage], DictPrompt]: model_config = self.model_config tokenizer = self.get_tokenizer() @@ -1315,6 +1188,7 @@ async def render_messages_async( ), media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, + defer_uuid_media=defer_uuid_media, ) prompt_embeds_tensors: list[torch.Tensor] | None = None diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index 51e17c98835e..ab9f78fe2f40 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -96,7 +96,7 @@ class ChatParams: """The kwargs to pass to the multi-modal processor.""" skip_early_mm_lookup: bool = False - """Skip the UUID-only rendering attempt.""" + """Opt out of deferring media loading to a processor cache lookup.""" tool_choice: Any | None = None """Request-level tool choice for renderers that need API metadata.""" From b8b67c769f8ea0b2959981d6ce79ac2b2ec8b2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Kwieci=C5=84ski?= <153385549+Jankwi@users.noreply.github.com> Date: Fri, 23 Oct 2026 15:02:17 +0200 Subject: [PATCH 9/9] Minor changes (renames, tests, reorganization) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Kwieciński <153385549+Jankwi@users.noreply.github.com> --- docs/features/multimodal_inputs.md | 9 +- tests/renderers/chat_utils/test_audio.py | 44 ---- .../renderers/chat_utils/test_chat_message.py | 71 +++++++ .../test_process_multi_modal_uuids.py | 196 ++++++------------ vllm/renderers/base.py | 91 ++++---- vllm/renderers/chat_utils/__init__.py | 4 +- vllm/renderers/chat_utils/chat_message.py | 20 +- vllm/renderers/chat_utils/content_parser.py | 26 +-- vllm/renderers/chat_utils/item_tracker.py | 10 +- vllm/renderers/hf.py | 22 +- vllm/renderers/params.py | 2 +- 11 files changed, 234 insertions(+), 261 deletions(-) create mode 100644 tests/renderers/chat_utils/test_chat_message.py diff --git a/docs/features/multimodal_inputs.md b/docs/features/multimodal_inputs.md index b95ac91260a4..6fcf6accc50f 100644 --- a/docs/features/multimodal_inputs.md +++ b/docs/features/multimodal_inputs.md @@ -1417,15 +1417,16 @@ Just like with offline inference, you can skip sending media if you expect cache ``` -#### UUID-Only Rendering +#### Deferred Media Fetching -The HF renderer first attempts to render a conversation without loading media when +The HF renderer first attempts to render a conversation without fetching media when every media item has a UUID and at least one image, audio, or video item is supplied as a URL or inline data. This is enabled by default. Set `skip_early_mm_lookup=True` to opt out per request. -- If every media item is in the processor cache, loading and decoding are skipped. -- If any item misses, vLLM retries that conversation once, loading the supplied media +- If every media item is in the processor cache, fetching (downloading and decoding) + is skipped. +- If any item misses, vLLM retries that conversation once, fetching the supplied media normally. Other conversations in the batch are not retried. This optimization requires the processor cache and is disabled for unified vision diff --git a/tests/renderers/chat_utils/test_audio.py b/tests/renderers/chat_utils/test_audio.py index 392d4355a055..bbfe4ca8de81 100644 --- a/tests/renderers/chat_utils/test_audio.py +++ b/tests/renderers/chat_utils/test_audio.py @@ -1,12 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import asyncio -from unittest.mock import AsyncMock, Mock - import pytest from vllm.config import ModelConfig -from vllm.multimodal.media import MediaConnector from vllm.renderers.chat_utils import parse_chat_messages, parse_chat_messages_async from vllm.utils.serial_utils import tensor2base64 @@ -226,43 +222,3 @@ async def test_parse_chat_messages_audio_embeds_async( assert mm_uuids is not None assert "audio" in mm_uuids assert_mm_uuids(mm_uuids, 1, modality="audio", expected_uuids=[None]) - - -@pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) -@pytest.mark.parametrize( - "part", - [ - {"type": "audio_url", "audio_url": {"url": "https://example.com/a.wav"}}, - {"type": "input_audio", "input_audio": {"data": "AAAA", "format": "wav"}}, - ], - ids=["audio_url", "input_audio"], -) -def test_parse_chat_messages_defers_audio_with_uuid( - qwen2_audio_model_config, monkeypatch, use_async, part -): - """Audio with a UUID is not loaded when deferring to a cache lookup.""" - fetch_audio = AsyncMock() if use_async else Mock() - method = "fetch_audio_async" if use_async else "fetch_audio" - monkeypatch.setattr(MediaConnector, method, fetch_audio) - messages = [ - { - "role": "user", - "content": [ - {**part, "uuid": "abcd"}, - {"type": "text", "text": "What does the audio say?"}, - ], - } - ] - - parse = parse_chat_messages_async if use_async else parse_chat_messages - result = parse( - messages, - qwen2_audio_model_config, - content_format="string", - defer_uuid_media=True, - ) - _, mm_data, mm_uuids = asyncio.run(result) if use_async else result - - fetch_audio.assert_not_called() - assert mm_data == {"audio": [None]} - assert mm_uuids == {"audio": ["abcd"]} diff --git a/tests/renderers/chat_utils/test_chat_message.py b/tests/renderers/chat_utils/test_chat_message.py new file mode 100644 index 000000000000..e150db17a998 --- /dev/null +++ b/tests/renderers/chat_utils/test_chat_message.py @@ -0,0 +1,71 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest + +from vllm.renderers.chat_utils import ( + can_conversation_defer_media_fetching, + has_non_text_content, +) + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + pytest.param("Describe", False, id="text"), + pytest.param( + [{"type": "image_url", "image_url": {"url": "url"}}], True, id="image" + ), + ], +) +def test_has_non_text_content(content, expected): + assert has_non_text_content([{"role": "user", "content": content}]) is expected + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + pytest.param( + [{"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"}], + True, + id="image-url", + ), + pytest.param( + [ + {"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"}, + {"type": "video_url", "video_url": {"url": "url"}}, + ], + False, + id="missing-uuid", + ), + pytest.param( + [{"type": "image_url", "image_url": None, "uuid": "image"}], + False, + id="uuid-without-data", + ), + pytest.param( + [ + { + "type": "input_audio", + "input_audio": {"data": "", "format": "wav"}, + "uuid": "audio", + } + ], + False, + id="input-audio-without-data", + ), + ], +) +def test_can_conversation_defer_media_fetching(content, expected): + """Fetching is deferred only if all media has UUIDs and some has data.""" + messages = [{"role": "user", "content": content}] + + assert can_conversation_defer_media_fetching(messages) is expected + + +def test_can_conversation_defer_media_fetching_does_not_consume_iterators(): + part = {"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"} + content = iter([part]) + + messages = [{"role": "user", "content": content}] + assert not can_conversation_defer_media_fetching(messages) + assert next(content) == part diff --git a/tests/renderers/test_process_multi_modal_uuids.py b/tests/renderers/test_process_multi_modal_uuids.py index 5d8fb98a52d4..7ca8f0856d15 100644 --- a/tests/renderers/test_process_multi_modal_uuids.py +++ b/tests/renderers/test_process_multi_modal_uuids.py @@ -8,6 +8,7 @@ import pytest +from vllm.assets.audio import AudioAsset from vllm.assets.image import ImageAsset from vllm.assets.video import VideoAsset from vllm.config import CacheConfig, ModelConfig, VllmConfig @@ -15,7 +16,6 @@ from vllm.multimodal.media import MediaConnector from vllm.multimodal.parse import parse_mm_uuids from vllm.multimodal.processing.processor import MultiModalProcessorCacheMissError -from vllm.renderers.chat_utils import can_conversation_defer_media from vllm.renderers.hf import HfRenderer from vllm.renderers.params import ChatParams from vllm.tokenizers.registry import cached_tokenizer_from_config @@ -27,13 +27,20 @@ baby_reading_video_asset.np_ndarrays, baby_reading_video_asset.metadata, ) +mary_had_lamb_audio = AudioAsset("mary_had_lamb").audio_and_sample_rate + +QWEN25_VL_MODEL_ID = "Qwen/Qwen2.5-VL-3B-Instruct" +QWEN25_OMNI_MODEL_ID = "Qwen/Qwen2.5-Omni-7B" def _build_renderer( - *, mm_cache_gb: float = 4.0, enable_prefix_caching: bool = True + *, + model: str = QWEN25_VL_MODEL_ID, + mm_cache_gb: float = 4.0, + enable_prefix_caching: bool = True, ) -> HfRenderer: model_config = ModelConfig( - model="Qwen/Qwen2.5-VL-3B-Instruct", + model=model, max_model_len=128, mm_processor_cache_gb=mm_cache_gb, ) @@ -88,101 +95,76 @@ def _mock_media(monkeypatch, modality, media, use_async): return fetch +def _media_part(part_type, uuid): + if part_type == "input_audio": + media = {"data": "AAAA", "format": "wav"} + else: + media = {"url": f"https://example.com/{uuid}.{part_type.removesuffix('_url')}"} + return {"type": part_type, part_type: media, "uuid": uuid} + + def _media_messages(name, *items): - return [ - { - "role": "user", - "content": [{"type": "text", "text": name}] - + [ - { - "type": f"{modality}_url", - f"{modality}_url": { - "url": f"https://example.com/{uuid}.{modality}" - }, - "uuid": uuid, - } - for modality, uuid in items - ], - } - ] + content = [{"type": "text", "text": name}] + content += [_media_part(part_type, uuid) for part_type, uuid in items] + return [{"role": "user", "content": content}] @pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize( - ("modality", "media", "skip_early_mm_lookup"), + ("part_type", "modality", "media", "model"), [ - pytest.param("image", cherry_pil_image, False, id="image"), - pytest.param("video", baby_reading_video, False, id="video"), pytest.param( - "video", - baby_reading_video, - True, - id="video-skip-early-mm-lookup", + "image_url", "image", cherry_pil_image, QWEN25_VL_MODEL_ID, id="image" + ), + pytest.param( + "video_url", "video", baby_reading_video, QWEN25_VL_MODEL_ID, id="video" + ), + pytest.param( + "audio_url", "audio", mary_had_lamb_audio, QWEN25_OMNI_MODEL_ID, id="audio" + ), + pytest.param( + "input_audio", + "audio", + mary_had_lamb_audio, + QWEN25_OMNI_MODEL_ID, + id="input-audio", ), ], ) -def test_cached_uuid_skips_url_loading( +def test_cached_uuid_skips_media_fetching( monkeypatch: pytest.MonkeyPatch, + part_type: str, modality: str, media: object, - skip_early_mm_lookup: bool, + model: str, use_async: bool, ): - renderer = _build_renderer() + renderer = _build_renderer(model=model) fetch_media = _mock_media(monkeypatch, modality, media, use_async) - messages = _media_messages("Describe", (modality, f"test-{modality}-uuid")) + messages = _media_messages("Describe", (part_type, f"test-{modality}-uuid")) - params = ChatParams(skip_early_mm_lookup=skip_early_mm_lookup) - _, first_prompts = _render_chat(renderer, [messages], params, use_async) - _, second_prompts = _render_chat(renderer, [messages], params, use_async) + _, first_prompts = _render_chat(renderer, [messages], ChatParams(), use_async) + _, second_prompts = _render_chat(renderer, [messages], ChatParams(), use_async) first_input = first_prompts[0] second_input = second_prompts[0] assert first_input["mm_hashes"] == second_input["mm_hashes"] assert first_input["prompt_token_ids"] == second_input["prompt_token_ids"] assert first_input["mm_placeholders"] == second_input["mm_placeholders"] - assert fetch_media.call_count == (2 if skip_early_mm_lookup else 1) - - -@pytest.mark.parametrize( - ("modality", "media"), - [ - pytest.param("image", cherry_pil_image, id="image"), - pytest.param("video", baby_reading_video, id="video"), - ], -) -def test_uuid_cache_eviction_falls_back_to_url( - monkeypatch: pytest.MonkeyPatch, - modality: str, - media: object, -): - renderer = _build_renderer() - fetch_media = _mock_media(monkeypatch, modality, media, use_async=False) - messages = _media_messages("Describe", (modality, f"test-{modality}-uuid")) - - renderer.render_chat([messages], ChatParams()) - renderer.clear_mm_cache() - fetch_media.reset_mock() - - _, prompts = renderer.render_chat([messages], ChatParams()) - - assert len(prompts[0]["mm_hashes"][modality]) == 1 assert fetch_media.call_count == 1 @pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) def test_partial_uuid_hit_retries_only_affected_conversation(monkeypatch, use_async): - """A miss reloads all URLs in its conversation without rerendering a batch peer.""" + """A miss refetches all media in its conversation without rerendering a peer.""" renderer = _build_renderer() fetch_image = _mock_media(monkeypatch, "image", cherry_pil_image, use_async) fetch_video = _mock_media(monkeypatch, "video", baby_reading_video, use_async) - cached = _media_messages("Cached conversation", ("image", "cached-image")) - partial = _media_messages("Partial conversation", ("image", "partial-image")) + cached = _media_messages("Cached conversation", ("image_url", "cached-image")) + partial = _media_messages("Partial conversation", ("image_url", "partial-image")) params = ChatParams() _, warm_prompts = _render_chat(renderer, [cached, partial], params, use_async) - partial[0]["content"].append( - _media_messages("", ("video", "missing-video"))[0]["content"][1] - ) + partial[0]["content"].append(_media_part("video_url", "missing-video")) conversations = [cached, partial] original = deepcopy(conversations) fetch_image.reset_mock() @@ -205,67 +187,6 @@ def test_partial_uuid_hit_retries_only_affected_conversation(monkeypatch, use_as assert conversations == original -@pytest.mark.parametrize( - ("content", "expected"), - [ - pytest.param( - [{"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"}], - True, - id="image-url", - ), - pytest.param( - [ - { - "type": "input_audio", - "input_audio": {"data": "AAAA", "format": "wav"}, - "uuid": "audio", - } - ], - True, - id="input-audio", - ), - pytest.param( - [{"type": "image_url", "image_url": None, "uuid": "image"}], - False, - id="uuid-without-data", - ), - pytest.param( - [ - { - "type": "input_audio", - "input_audio": {"data": "", "format": "wav"}, - "uuid": "audio", - } - ], - False, - id="input-audio-without-data", - ), - pytest.param( - [ - {"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"}, - {"type": "video_url", "video_url": {"url": "url"}}, - ], - False, - id="missing-uuid", - ), - pytest.param("Describe", False, id="text-only"), - ], -) -def test_can_conversation_defer_media(content, expected): - """Media loading is deferred only if all media has UUIDs and some has data.""" - messages = [{"role": "user", "content": content}] - - assert can_conversation_defer_media(messages) is expected - - -def test_can_conversation_defer_media_does_not_consume_iterators(): - part = {"type": "image_url", "image_url": {"url": "url"}, "uuid": "image"} - content = iter([part]) - - assert not can_conversation_defer_media([{"role": "user", "content": content}]) - assert next(content) == part - - def _wrap_process_for_engine(monkeypatch, renderer, use_async): method = "process_for_engine_async" if use_async else "process_for_engine" mock_type = AsyncMock if use_async else Mock @@ -276,17 +197,17 @@ def _wrap_process_for_engine(monkeypatch, renderer, use_async): @pytest.mark.parametrize("use_async", [False, True], ids=["sync", "async"]) @pytest.mark.parametrize( - "case", ["skip-cache", "no-cache", "audio-in-video", "missing-uuid"] + "case", ["skip-cache", "no-cache", "opt-out", "audio-in-video"] ) -def test_ineligible_conversation_loads_media_upfront(monkeypatch, use_async, case): - """Conversations that cannot defer media are processed once, not retried.""" +def test_ineligible_request_fetches_media_upfront(monkeypatch, use_async, case): + """Requests that cannot defer media fetching are processed once, not retried.""" renderer = _build_renderer(mm_cache_gb=0 if case == "no-cache" else 4) fetch_image = _mock_media(monkeypatch, "image", cherry_pil_image, use_async) - messages = _media_messages("Describe", ("image", "image")) - if case == "missing-uuid": - del messages[0]["content"][1]["uuid"] + messages = _media_messages("Describe", ("image_url", "image")) params = ChatParams() - if case == "audio-in-video": + if case == "opt-out": + params = ChatParams(skip_early_mm_lookup=True) + elif case == "audio-in-video": params = ChatParams(mm_processor_kwargs={"use_audio_in_video": True}) process = _wrap_process_for_engine(monkeypatch, renderer, use_async) @@ -306,7 +227,7 @@ def test_ineligible_conversation_loads_media_upfront(monkeypatch, use_async, cas def test_uuid_only_cache_miss_is_not_retried(monkeypatch, use_async): """Without a URL to fall back to, a cache miss fails without a retry.""" renderer = _build_renderer() - messages = _media_messages("Describe", ("image", "image")) + messages = _media_messages("Describe", ("image_url", "image")) messages[0]["content"][1]["image_url"] = None process = _wrap_process_for_engine(monkeypatch, renderer, use_async) @@ -316,8 +237,8 @@ def test_uuid_only_cache_miss_is_not_retried(monkeypatch, use_async): assert process.call_count == 1 -def test_batch_cache_misses_reload_media_concurrently(monkeypatch): - """Retried conversations load media concurrently and keep the batch order.""" +def test_batch_cache_misses_refetch_media_concurrently(monkeypatch): + """Retried conversations fetch media concurrently and keep the batch order.""" renderer = _build_renderer() active_fetches = max_active_fetches = 0 @@ -333,7 +254,8 @@ async def fetch_image(*args, **kwargs): monkeypatch.setattr(MediaConnector, "fetch_image_async", fetch_image) conversations = [ - _media_messages(f"Conversation {i}", ("image", f"image-{i}")) for i in range(2) + _media_messages(f"Conversation {i}", ("image_url", f"image-{i}")) + for i in range(2) ] out_conversations, prompts = _render_chat( diff --git a/vllm/renderers/base.py b/vllm/renderers/base.py index 6934c586fc28..8cc21421b101 100644 --- a/vllm/renderers/base.py +++ b/vllm/renderers/base.py @@ -53,7 +53,7 @@ from vllm.utils.torch_utils import set_default_torch_num_threads from vllm.v1.metrics.stats import MultiModalCacheStats -from .chat_utils import can_conversation_defer_media, uses_audio_in_video +from .chat_utils import can_conversation_defer_media_fetching, uses_audio_in_video from .embed_utils import safe_load_prompt_embeds from .inputs import ( DictPrompt, @@ -1294,30 +1294,30 @@ async def render_cmpl_async( ) ) - def _can_defer_uuid_media(self) -> bool: - """Whether this renderer implements `_render_messages_deferring_media`.""" + def _can_defer_uuid_media_fetching(self) -> bool: + """Whether this renderer can render messages without fetching UUID media.""" return False - def _render_messages_deferring_media( + def _render_messages_deferring_uuid_media_fetching( self, messages: list["ChatCompletionMessageParam"], params: ChatParams, ) -> tuple[list["ConversationMessage"], DictPrompt]: - """Like `render_messages`, but without loading media that has a UUID.""" + """Like `render_messages`, but without fetching media that has a UUID.""" raise NotImplementedError - async def _render_messages_deferring_media_async( + async def _render_messages_deferring_uuid_media_fetching_async( self, messages: list["ChatCompletionMessageParam"], params: ChatParams, ) -> tuple[list["ConversationMessage"], DictPrompt]: - return self._render_messages_deferring_media(messages, params) + return self._render_messages_deferring_uuid_media_fetching(messages, params) - def _should_defer_uuid_media( + def _should_defer_uuid_media_fetching( self, chat_params: ChatParams, skip_mm_cache: bool ) -> bool: return ( - self._can_defer_uuid_media() + self._can_defer_uuid_media_fetching() and self._mm_processor_cache is not None and not skip_mm_cache and not chat_params.skip_early_mm_lookup @@ -1331,24 +1331,30 @@ def _render_chat_prompts( tok_params: TokenizeParams, prompt_extras: dict[str, Any], *, - defer_uuid_media: bool, + defer_uuid_media_fetching: bool, ) -> tuple[list[list["ConversationMessage"]], list[TokPrompt], list[bool]]: """Render and tokenize `conversations`. Returns: The rendered conversations, their prompts, and whether each prompt - had its media loading deferred. + had its media fetching deferred. """ def render(conversation: list["ChatCompletionMessageParam"]): - deferred = defer_uuid_media and can_conversation_defer_media(conversation) + is_media_fetching_deferred = ( + defer_uuid_media_fetching + and can_conversation_defer_media_fetching(conversation) + ) render_messages = ( - self._render_messages_deferring_media - if deferred + self._render_messages_deferring_uuid_media_fetching + if is_media_fetching_deferred else self.render_messages ) - return *render_messages(conversation, chat_params), deferred + return ( + *render_messages(conversation, chat_params), + is_media_fetching_deferred, + ) rendered = [render(conversation) for conversation in conversations] @@ -1360,7 +1366,7 @@ def render(conversation: list["ChatCompletionMessageParam"]): return ( [conv for conv, _, _ in rendered], tok_prompts, - [deferred for _, _, deferred in rendered], + [is_deferred for _, _, is_deferred in rendered], ) async def _render_chat_prompts_async( @@ -1370,16 +1376,22 @@ async def _render_chat_prompts_async( tok_params: TokenizeParams, prompt_extras: dict[str, Any], *, - defer_uuid_media: bool, + defer_uuid_media_fetching: bool, ) -> tuple[list[list["ConversationMessage"]], list[TokPrompt], list[bool]]: async def render(conversation: list["ChatCompletionMessageParam"]): - deferred = defer_uuid_media and can_conversation_defer_media(conversation) + is_media_fetching_deferred = ( + defer_uuid_media_fetching + and can_conversation_defer_media_fetching(conversation) + ) render_messages = ( - self._render_messages_deferring_media_async - if deferred + self._render_messages_deferring_uuid_media_fetching_async + if is_media_fetching_deferred else self.render_messages_async ) - return *await render_messages(conversation, chat_params), deferred + return ( + *await render_messages(conversation, chat_params), + is_media_fetching_deferred, + ) rendered = await asyncio.gather(*(render(c) for c in conversations)) @@ -1391,7 +1403,7 @@ async def render(conversation: list["ChatCompletionMessageParam"]): return ( [conv for conv, _, _ in rendered], tok_prompts, - [deferred for _, _, deferred in rendered], + [is_deferred for _, _, is_deferred in rendered], ) def render_chat( @@ -1411,15 +1423,18 @@ def render_chat( prompt_extras = dict(prompt_extras or {}) prompt_extras["media_io_kwargs"] = chat_params.media_io_kwargs or {} - out_conversations, tok_prompts, deferred = self._render_chat_prompts( + rendered = self._render_chat_prompts( conversations, chat_params, tok_params, prompt_extras, - defer_uuid_media=self._should_defer_uuid_media(chat_params, skip_mm_cache), + defer_uuid_media_fetching=self._should_defer_uuid_media_fetching( + chat_params, skip_mm_cache + ), ) + out_conversations, tok_prompts, media_fetching_deferred = rendered - def process(prompt: TokPrompt, is_deferred: bool = False): + def process(prompt: TokPrompt, is_media_fetching_deferred: bool = False): try: return self.process_for_engine( prompt, @@ -1428,20 +1443,22 @@ def process(prompt: TokPrompt, is_deferred: bool = False): tok_params=tok_params, ) except MultiModalProcessorCacheMissError: - if not is_deferred: + if not is_media_fetching_deferred: raise return None - eng_prompts = [process(p, d) for p, d in zip(tok_prompts, deferred)] + eng_prompts = [ + process(p, d) for p, d in zip(tok_prompts, media_fetching_deferred) + ] - # Retry the deferred prompts that missed the cache with their media loaded. + # Retry the deferred prompts that missed the cache with their media fetched. if missed := [i for i, prompt in enumerate(eng_prompts) if prompt is None]: retry_conversations, retry_prompts, _ = self._render_chat_prompts( [conversations[i] for i in missed], chat_params, tok_params, prompt_extras, - defer_uuid_media=False, + defer_uuid_media_fetching=False, ) for i, conv, prompt in zip(missed, retry_conversations, retry_prompts): out_conversations[i] = conv @@ -1471,11 +1488,13 @@ async def render_chat_async( chat_params, tok_params, prompt_extras, - defer_uuid_media=self._should_defer_uuid_media(chat_params, skip_mm_cache), + defer_uuid_media_fetching=self._should_defer_uuid_media_fetching( + chat_params, skip_mm_cache + ), ) - out_conversations, tok_prompts, deferred = rendered + out_conversations, tok_prompts, media_fetching_deferred = rendered - async def process(prompt: TokPrompt, is_deferred: bool = False): + async def process(prompt: TokPrompt, is_media_fetching_deferred: bool = False): try: return await self.process_for_engine_async( prompt, @@ -1484,22 +1503,22 @@ async def process(prompt: TokPrompt, is_deferred: bool = False): tok_params=tok_params, ) except MultiModalProcessorCacheMissError: - if not is_deferred: + if not is_media_fetching_deferred: raise return None eng_prompts = await asyncio.gather( - *(process(p, d) for p, d in zip(tok_prompts, deferred)) + *(process(p, d) for p, d in zip(tok_prompts, media_fetching_deferred)) ) - # Retry the deferred prompts that missed the cache with their media loaded. + # Retry the deferred prompts that missed the cache with their media fetched. if missed := [i for i, prompt in enumerate(eng_prompts) if prompt is None]: retried = await self._render_chat_prompts_async( [conversations[i] for i in missed], chat_params, tok_params, prompt_extras, - defer_uuid_media=False, + defer_uuid_media_fetching=False, ) retry_conversations, retry_prompts, _ = retried retry_eng_prompts = await asyncio.gather( diff --git a/vllm/renderers/chat_utils/__init__.py b/vllm/renderers/chat_utils/__init__.py index e418ef963c06..c62081cfb04f 100644 --- a/vllm/renderers/chat_utils/__init__.py +++ b/vllm/renderers/chat_utils/__init__.py @@ -2,7 +2,7 @@ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from .chat_message import ( - can_conversation_defer_media, + can_conversation_defer_media_fetching, has_non_text_content, parse_chat_message_content, parse_chat_message_content_parts, @@ -34,7 +34,7 @@ "BaseMultiModalItemTracker", "MultiModalItemTracker", "AsyncMultiModalItemTracker", - "can_conversation_defer_media", + "can_conversation_defer_media_fetching", "has_non_text_content", "parse_chat_message_content", "parse_chat_message_content_parts", diff --git a/vllm/renderers/chat_utils/chat_message.py b/vllm/renderers/chat_utils/chat_message.py index e6122f1b62aa..a21cb95a44f3 100644 --- a/vllm/renderers/chat_utils/chat_message.py +++ b/vllm/renderers/chat_utils/chat_message.py @@ -345,8 +345,8 @@ def parse_chat_message_content_mm_part( _TEXT_CONTENT_PART_TYPES = TEXT_PART_TYPES | {"tool_reference"} # Keys that mark a content part as multimodal, whatever its ``type``. _MEDIA_CONTENT_PART_KEYS = frozenset(MM_PARSER_MAP) - _TEXT_CONTENT_PART_TYPES -# Content part types whose media is loaded from a URL or inline data. -_LOADED_MEDIA_PART_TYPES = frozenset( +# Content part types whose media is fetched from a URL or inline data. +_FETCHED_MEDIA_PART_TYPES = frozenset( {"image_url", "audio_url", "input_audio", "video_url"} ) @@ -376,9 +376,11 @@ def has_non_text_content(messages: Any) -> bool: return False -def can_conversation_defer_media(messages: list[ChatCompletionMessageParam]) -> bool: +def can_conversation_defer_media_fetching( + messages: list[ChatCompletionMessageParam], +) -> bool: """Whether every media part in ``messages`` has a UUID and at least one - has data to load. + has data to fetch. Only list content is inspected, so content that is a one-shot iterator is never consumed; messages with such content cannot defer. @@ -396,7 +398,7 @@ def can_conversation_defer_media(messages: list[ChatCompletionMessageParam]) -> if part.get("uuid") is None: return False part_type, data = parse_chat_message_content_mm_part(part) - if part_type in _LOADED_MEDIA_PART_TYPES: + if part_type in _FETCHED_MEDIA_PART_TYPES: # `input_audio` carries its data inside a dict. has_data |= bool(data.get("data") if isinstance(data, dict) else data) return has_data @@ -785,7 +787,7 @@ def parse_chat_messages( content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, - defer_uuid_media: bool = False, + defer_uuid_media_fetching: bool = False, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -795,7 +797,7 @@ def parse_chat_messages( mm_tracker = MultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, - defer_uuid_media=defer_uuid_media, + defer_uuid_media_fetching=defer_uuid_media_fetching, ) for msg in messages: @@ -826,7 +828,7 @@ async def parse_chat_messages_async( content_format: ChatTemplateContentFormat, media_io_kwargs: dict[str, dict[str, Any]] | None = None, mm_processor_kwargs: dict[str, Any] | None = None, - defer_uuid_media: bool = False, + defer_uuid_media_fetching: bool = False, ) -> tuple[ list[ConversationMessage], MultiModalDataDict | None, @@ -836,7 +838,7 @@ async def parse_chat_messages_async( mm_tracker = AsyncMultiModalItemTracker( model_config, media_io_kwargs=media_io_kwargs, - defer_uuid_media=defer_uuid_media, + defer_uuid_media_fetching=defer_uuid_media_fetching, ) for msg in messages: diff --git a/vllm/renderers/chat_utils/content_parser.py b/vllm/renderers/chat_utils/content_parser.py index bddc6f6905f7..f1a886c9f0a6 100644 --- a/vllm/renderers/chat_utils/content_parser.py +++ b/vllm/renderers/chat_utils/content_parser.py @@ -54,8 +54,6 @@ def uses_audio_in_video(mm_processor_kwargs: dict[str, Any] | None) -> bool: class BaseMultiModalContentParser(ABC): - _mm_processor_kwargs: dict[str, Any] | None - def __init__(self) -> None: super().__init__() @@ -81,10 +79,6 @@ def _add_placeholder(self, modality: ModalityStr, placeholder: str | None): def mm_placeholder_storage(self) -> dict[str, list]: return dict(self._placeholder_storage) - @cached_property - def _use_audio_in_video(self) -> bool: - return uses_audio_in_video(self._mm_processor_kwargs) - @abstractmethod def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: raise NotImplementedError @@ -162,6 +156,10 @@ def _connector(self) -> MediaConnector: download_budget=self._tracker.download_budget, ) + @cached_property + def _use_audio_in_video(self) -> bool: + return uses_audio_in_video(self._mm_processor_kwargs) + @property def model_config(self) -> ModelConfig: return self._tracker.model_config @@ -184,7 +182,7 @@ def parse_prompt_embeds(self, data: str) -> None: self._add_placeholder("prompt_embeds", PROMPT_EMBEDS_PLACEHOLDER_TOKEN) def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: - image_url = self._tracker.defer_media_url(image_url, uuid) + image_url = self._tracker.url_to_fetch(image_url, uuid) image = self._connector.fetch_image(image_url) if image_url else None placeholder = self._tracker.add("image", (image, uuid)) @@ -255,7 +253,7 @@ def parse_image_pil( self._add_placeholder("image", placeholder) def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None: - audio_url = self._tracker.defer_media_url(audio_url, uuid) + audio_url = self._tracker.url_to_fetch(audio_url, uuid) audio = self._connector.fetch_audio(audio_url) if audio_url else None placeholder = self._tracker.add("audio", (audio, uuid)) @@ -278,7 +276,7 @@ def parse_input_audio( return self.parse_audio(audio_url, uuid) def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: - video_url = self._tracker.defer_media_url(video_url, uuid) + video_url = self._tracker.url_to_fetch(video_url, uuid) video = ( self._connector.fetch_video( video_url=video_url, @@ -348,6 +346,10 @@ def _connector(self) -> MediaConnector: download_budget=self._tracker.download_budget, ) + @cached_property + def _use_audio_in_video(self) -> bool: + return uses_audio_in_video(self._mm_processor_kwargs) + @property def model_config(self) -> ModelConfig: return self._tracker.model_config @@ -388,7 +390,7 @@ async def _image_with_uuid_async(self, image_url: str | None, uuid: str | None): return image, uuid def parse_image(self, image_url: str | None, uuid: str | None = None) -> None: - image_url = self._tracker.defer_media_url(image_url, uuid) + image_url = self._tracker.url_to_fetch(image_url, uuid) placeholder = self._tracker.add( "image", partial(self._image_with_uuid_async, image_url, uuid) ) @@ -481,7 +483,7 @@ async def _audio_with_uuid_async(self, audio_url: str | None, uuid: str | None): return audio, uuid def parse_audio(self, audio_url: str | None, uuid: str | None = None) -> None: - audio_url = self._tracker.defer_media_url(audio_url, uuid) + audio_url = self._tracker.url_to_fetch(audio_url, uuid) placeholder = self._tracker.add( "audio", partial(self._audio_with_uuid_async, audio_url, uuid) ) @@ -515,7 +517,7 @@ async def _video_with_uuid_async(self, video_url: str | None, uuid: str | None): return video, uuid def parse_video(self, video_url: str | None, uuid: str | None = None) -> None: - video_url = self._tracker.defer_media_url(video_url, uuid) + video_url = self._tracker.url_to_fetch(video_url, uuid) placeholder = self._tracker.add( "video", partial(self._video_with_uuid_async, video_url, uuid) ) diff --git a/vllm/renderers/chat_utils/item_tracker.py b/vllm/renderers/chat_utils/item_tracker.py index 11e7f8502142..78e3a2c4d5a8 100644 --- a/vllm/renderers/chat_utils/item_tracker.py +++ b/vllm/renderers/chat_utils/item_tracker.py @@ -207,13 +207,13 @@ def __init__( self, model_config: ModelConfig, media_io_kwargs: dict[str, dict[str, Any]] | None = None, - defer_uuid_media: bool = False, + defer_uuid_media_fetching: bool = False, ): super().__init__() self._model_config = model_config self._media_io_kwargs = media_io_kwargs - self._defer_uuid_media = defer_uuid_media + self._defer_uuid_media_fetching = defer_uuid_media_fetching self._items_by_modality = defaultdict[str, list[_T]](list) # Track original modality for each vision_chunk item (image or video) @@ -265,9 +265,9 @@ def mm_processor(self): def video_processor_name(self) -> str | None: return get_video_processor_cls_name(self.model_config) - def defer_media_url(self, url: str | None, uuid: str | None) -> str | None: - """Return `None` in place of `url` if loading media with a UUID is deferred.""" - return None if self._defer_uuid_media and uuid is not None else url + def url_to_fetch(self, url: str | None, uuid: str | None) -> str | None: + """Return `url`, or `None` if fetching media with a UUID is deferred.""" + return None if self._defer_uuid_media_fetching and uuid is not None else url def add(self, modality: ModalityStr, item: _T) -> str | None: """Add a multi-modal item to the current prompt and returns the diff --git a/vllm/renderers/hf.py b/vllm/renderers/hf.py index 68deb5875c4f..545c24f207c1 100644 --- a/vllm/renderers/hf.py +++ b/vllm/renderers/hf.py @@ -1013,7 +1013,7 @@ def _can_produce_offsets(self) -> bool: return self.tokenizer is not None and self.tokenizer.is_fast @override - def _can_defer_uuid_media(self) -> bool: + def _can_defer_uuid_media_fetching(self) -> bool: return not self.use_unified_vision_chunk def render_messages( @@ -1021,22 +1021,22 @@ def render_messages( messages: list[ChatCompletionMessageParam], params: ChatParams, ) -> tuple[list[ConversationMessage], DictPrompt]: - return self._render_messages(messages, params, defer_uuid_media=False) + return self._render_messages(messages, params, defer_uuid_media_fetching=False) @override - def _render_messages_deferring_media( + def _render_messages_deferring_uuid_media_fetching( self, messages: list[ChatCompletionMessageParam], params: ChatParams, ) -> tuple[list[ConversationMessage], DictPrompt]: - return self._render_messages(messages, params, defer_uuid_media=True) + return self._render_messages(messages, params, defer_uuid_media_fetching=True) def _render_messages( self, messages: list[ChatCompletionMessageParam], params: ChatParams, *, - defer_uuid_media: bool, + defer_uuid_media_fetching: bool, ) -> tuple[list[ConversationMessage], DictPrompt]: model_config = self.model_config tokenizer = self.get_tokenizer() @@ -1059,7 +1059,7 @@ def _render_messages( ), media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, - defer_uuid_media=defer_uuid_media, + defer_uuid_media_fetching=defer_uuid_media_fetching, ) # prompt_embeds tensors are carried by the tracker through mm_data, @@ -1147,17 +1147,17 @@ async def render_messages_async( params: ChatParams, ) -> tuple[list[ConversationMessage], DictPrompt]: return await self._render_messages_async( - messages, params, defer_uuid_media=False + messages, params, defer_uuid_media_fetching=False ) @override - async def _render_messages_deferring_media_async( + async def _render_messages_deferring_uuid_media_fetching_async( self, messages: list[ChatCompletionMessageParam], params: ChatParams, ) -> tuple[list[ConversationMessage], DictPrompt]: return await self._render_messages_async( - messages, params, defer_uuid_media=True + messages, params, defer_uuid_media_fetching=True ) async def _render_messages_async( @@ -1165,7 +1165,7 @@ async def _render_messages_async( messages: list[ChatCompletionMessageParam], params: ChatParams, *, - defer_uuid_media: bool, + defer_uuid_media_fetching: bool, ) -> tuple[list[ConversationMessage], DictPrompt]: model_config = self.model_config tokenizer = self.get_tokenizer() @@ -1188,7 +1188,7 @@ async def _render_messages_async( ), media_io_kwargs=params.media_io_kwargs, mm_processor_kwargs=params.mm_processor_kwargs, - defer_uuid_media=defer_uuid_media, + defer_uuid_media_fetching=defer_uuid_media_fetching, ) prompt_embeds_tensors: list[torch.Tensor] | None = None diff --git a/vllm/renderers/params.py b/vllm/renderers/params.py index ab9f78fe2f40..adbac5494eb9 100644 --- a/vllm/renderers/params.py +++ b/vllm/renderers/params.py @@ -96,7 +96,7 @@ class ChatParams: """The kwargs to pass to the multi-modal processor.""" skip_early_mm_lookup: bool = False - """Opt out of deferring media loading to a processor cache lookup.""" + """Fetch media with UUIDs upfront instead of first trying the processor cache.""" tool_choice: Any | None = None """Request-level tool choice for renderers that need API metadata."""