diff --git a/Cargo.lock b/Cargo.lock index f92d5a02b9ce..ce30ee523073 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,7 +121,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -132,7 +132,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1563,7 +1563,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -2302,7 +2302,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2722,9 +2722,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18f8f6a2d325675a65beb0718940124a011e5ea25ee827df19028b965296a6c" +checksum = "bf45340e7f5beb8f3055c1fb933394f05256b6548a1b1434ae3df27bfc8d2f6e" dependencies = [ "aho-corasick", "anyhow", @@ -2760,9 +2760,9 @@ dependencies = [ [[package]] name = "dynamo-protocols" -version = "2.0.2" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1da866e573424e374c83d64dbf7807547054ebadcd85ddb2f9f67db06bf386" +checksum = "aaf9875c31601458f29b83e5ed0bf63479dc3a086e4131bbee0863101f4fd5f3" dependencies = [ "async-openai", "derive_builder", @@ -2777,9 +2777,9 @@ dependencies = [ [[package]] name = "dynamo-renderer" -version = "1.3.8" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3472f4ee978f581bac985c4f5ebad55a9b1b4ccb025ed3e9df708efa56e83bfa" +checksum = "1afd1dc880ea81af098f1d0586af1ab37865c82f87d14265b7e72bf00696ffef" dependencies = [ "anyhow", "chrono", @@ -2906,9 +2906,9 @@ dependencies = [ [[package]] name = "dynamo-tokenizers" -version = "1.5.0" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb9c031ea19d99d1faf8ff304fe666eb2dbd4666fcb416af3f08ae88f5be80" +checksum = "2843cd9cd50002ae23f3df525e1c551775b7bc7c9041742539faf63bc35ea225" dependencies = [ "aho-corasick", "anyhow", @@ -3144,7 +3144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4114,7 +4114,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration 0.6.1", "tokio", "tower-service", @@ -4450,7 +4450,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5827,7 +5827,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6990,8 +6990,8 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -7010,8 +7010,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -7032,7 +7032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -7045,7 +7045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -7235,7 +7235,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls 0.23.40", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -7272,7 +7272,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.60.2", ] @@ -7919,7 +7919,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8696,7 +8696,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8962,7 +8962,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10786,7 +10786,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 82d520f74112..c7a07701f7fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,8 +46,8 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed"] dynamo-runtime = { path = "lib/runtime", version = "1.3.0" } dynamo-llm = { path = "lib/llm", version = "1.3.0" } dynamo-rl = { path = "lib/rl", version = "1.3.0" } -dynamo-tokenizers = { version = "=1.5.0" } -dynamo-renderer = { version = "=1.3.8" } +dynamo-tokenizers = { version = "=1.5.3" } +dynamo-renderer = { version = "=2.0.0" } dynamo-tokens = { path = "lib/tokens", version = "1.3.0" } dynamo-truthy = { path = "lib/truthy", version = "1.3.0" } dynamo-data-gen = { path = "lib/data-gen", version = "1.3.0" } @@ -55,8 +55,8 @@ dynamo-kv-hashing = { path = "lib/kv-hashing", version = "1.3.0" } dynamo-memory = { path = "lib/memory", version = "1.3.0" } dynamo-mocker = { path = "lib/mocker", version = "1.3.0" } dynamo-kv-router = { path = "lib/kv-router", version = "1.3.0" } -dynamo-protocols = { version = "=2.0.2" } -dynamo-parsers = { version = "5.0.0" } +dynamo-protocols = { version = "3.0.0" } +dynamo-parsers = { version = "5.1.0" } # Published on crates.io. To see all available versions: # web: https://crates.io/crates/dynamo-parsers-v2/versions # sparse index: curl https://index.crates.io/dy/na/dynamo-parsers-v2 (one JSON line per release; read the "vers" field) diff --git a/components/src/dynamo/common/multimodal/cache_uuid.py b/components/src/dynamo/common/multimodal/cache_uuid.py new file mode 100644 index 000000000000..ae2c9bef9e03 --- /dev/null +++ b/components/src/dynamo/common/multimodal/cache_uuid.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Backend capability guard for client-provided multimodal cache UUIDs.""" + + +def reject_unsupported_multimodal_uuids( + multi_modal_uuids: object, + *, + backend: str, +) -> None: + if not multi_modal_uuids: + return + + raise ValueError( + "Image UUID caching is currently supported only by the vLLM " + f"backend; {backend} does not support the `uuid` field" + ) diff --git a/components/src/dynamo/common/multimodal/image_loader.py b/components/src/dynamo/common/multimodal/image_loader.py index c23d5c9e25f5..ca4060b8670f 100644 --- a/components/src/dynamo/common/multimodal/image_loader.py +++ b/components/src/dynamo/common/multimodal/image_loader.py @@ -8,7 +8,7 @@ import os from collections import OrderedDict from io import BytesIO -from typing import Any, Dict, Final, List +from typing import Any, Coroutine, Dict, Final, List from urllib.parse import urlparse from PIL import Image @@ -28,6 +28,7 @@ # Constants for multimodal data variants URL_VARIANT_KEY: Final = "Url" DECODED_VARIANT_KEY: Final = "Decoded" +UUID_ONLY_VARIANT_KEY: Final = "UuidOnly" def _create_nixl_connector() -> Any: @@ -267,12 +268,14 @@ async def load_image_batch( Exception: If any image fails to load for any other reason ValueError: If enable_frontend_decoding=True but nixl_connector is None """ - image_futures = [] + image_futures: list[Coroutine[Any, Any, Image.Image]] = [] + slot_to_future_idx: list[int | None] = [] - for item in image_mm_items: + for idx, item in enumerate(image_mm_items): if isinstance(item, dict) and URL_VARIANT_KEY in item: # URL path: download and decode in Python backend url = item[URL_VARIANT_KEY] + slot_to_future_idx.append(len(image_futures)) image_futures.append(self.load_image(url)) logger.debug(f"Preparing to load image from URL: {url[:80]}...") elif isinstance(item, dict) and DECODED_VARIANT_KEY in item: @@ -280,6 +283,7 @@ async def load_image_batch( metadata = item[DECODED_VARIANT_KEY] if self._nixl_connector is None: raise RuntimeError("NIXL connector is not initialized") + slot_to_future_idx.append(len(image_futures)) image_futures.append(self._read_and_convert_nixl_image(metadata)) else: logger.error( @@ -287,15 +291,34 @@ async def load_image_batch( "Set enable_frontend_decoding=True to enable NIXL RDMA image transfer." ) raise ValueError("Could not load decoded media from frontend") + elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item: + slot_to_future_idx.append(None) + else: + raise ValueError( + f"Invalid image multimodal item at index {idx}. " + "Expected dict with 'Url', 'Decoded', or 'UuidOnly' key." + ) # Process images in parallel results = await asyncio.gather(*image_futures, return_exceptions=True) - loaded_images = [] + loaded_images: list[Image.Image | None] = [] collective_exceptions = "" status_error: HttpStatusError | None = None url_error: UrlValidationError | None = None - for media_item, result in zip(image_mm_items, results): - if isinstance(result, Exception): + for media_item, future_idx in zip( + image_mm_items, slot_to_future_idx, strict=True + ): + if future_idx is None: + loaded_images.append(None) + continue + result = results[future_idx] + if isinstance(result, BaseException): + # asyncio.gather(..., return_exceptions=True) may return a + # cancellation (a BaseException, but not an Exception). Keep + # cancellation semantics instead of folding it into a batch + # image-loading error. + if not isinstance(result, Exception): + raise result source = media_item.get(URL_VARIANT_KEY, "decoded") logger.error(f"Failed to load image from {source[:80]}...: {result}") collective_exceptions += ( diff --git a/components/src/dynamo/common/tests/multimodal/test_image_loader.py b/components/src/dynamo/common/tests/multimodal/test_image_loader.py index a745e3b07cb3..15d426ee75f7 100644 --- a/components/src/dynamo/common/tests/multimodal/test_image_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_image_loader.py @@ -260,6 +260,37 @@ async def test_unsupported_format_batch_url_raises_415(loader: ImageLoader) -> N assert exc_info.value.status == 415 +async def test_uuid_only_image_slots_preserve_batch_alignment( + loader: ImageLoader, +) -> None: + first = Image.new("RGB", (1, 1), color="red") + second = Image.new("RGB", (1, 1), color="blue") + loader.load_image = AsyncMock( # type: ignore[method-assign] + side_effect=[first, second] + ) + + images = await loader.load_image_batch( + [ + {"Url": "https://example.com/first.png"}, + {"UuidOnly": "cached-image"}, + {"Url": "https://example.com/second.png"}, + ] + ) + + assert images == [first, None, second] + + +async def test_batch_propagates_cancellation(loader: ImageLoader) -> None: + loader.load_image = AsyncMock( # type: ignore[method-assign] + side_effect=asyncio.CancelledError + ) + + with pytest.raises(asyncio.CancelledError): + await loader.load_image_batch( + [{URL_VARIANT_KEY: "https://example.com/image.png"}] + ) + + async def test_unsupported_format_batch_data_url_raises_415( loader: ImageLoader, ) -> None: diff --git a/components/src/dynamo/frontend/sglang_processor.py b/components/src/dynamo/frontend/sglang_processor.py index 2a7210b2b42a..e44abe18a635 100644 --- a/components/src/dynamo/frontend/sglang_processor.py +++ b/components/src/dynamo/frontend/sglang_processor.py @@ -20,6 +20,7 @@ from sglang.srt.utils.hf_transformers_utils import get_tokenizer from dynamo._internal import ModelDeploymentCard +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.frontend.frontend_args import FrontendConfig from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine from dynamo.llm.exceptions import InvalidArgument, Unknown @@ -327,7 +328,11 @@ def _build_dynamo_preproc( } # Forward multimodal URLs so the backend handler can load the media. - mm_data = extract_mm_urls(request.get("messages", [])) + mm_data, mm_uuids = extract_mm_urls(request.get("messages", [])) + try: + reject_unsupported_multimodal_uuids(mm_uuids, backend="SGLang") + except ValueError as exc: + raise PreprocessError(str(exc)) from exc if mm_data: preproc["multi_modal_data"] = mm_data @@ -579,7 +584,8 @@ async def _generate_and_stream( cumulative_output_tokens = 0 # Rust postprocessor is bypassed on this path, so emit the multimodal # content-part counts here too (else frontend metrics report zero media). - _mm_counts = extract_mm_urls(request.get("messages", [])) or {} + _mm_counts, _ = extract_mm_urls(request.get("messages", [])) + _mm_counts = _mm_counts or {} image_count = len(_mm_counts.get("image_url", [])) video_count = len(_mm_counts.get("video_url", [])) audio_count = len(_mm_counts.get("audio_url", [])) diff --git a/components/src/dynamo/frontend/tests/test_multimodal_utils.py b/components/src/dynamo/frontend/tests/test_multimodal_utils.py index 9946275a3a23..893ead26925e 100644 --- a/components/src/dynamo/frontend/tests/test_multimodal_utils.py +++ b/components/src/dynamo/frontend/tests/test_multimodal_utils.py @@ -16,11 +16,11 @@ def test_returns_none_for_text_only(): messages = [ {"role": "user", "content": "What is the capital of France?"}, ] - assert extract_mm_urls(messages) is None + assert extract_mm_urls(messages) == (None, None) def test_returns_none_for_empty_messages(): - assert extract_mm_urls([]) is None + assert extract_mm_urls([]) == (None, None) def test_extracts_image_urls(): @@ -37,7 +37,7 @@ def test_extracts_image_urls(): } ] result = extract_mm_urls(messages) - assert result == {"image_url": [{"Url": "https://example.com/cat.png"}]} + assert result == ({"image_url": [{"Url": "https://example.com/cat.png"}]}, None) def test_extracts_audio_urls(): @@ -54,7 +54,10 @@ def test_extracts_audio_urls(): } ] result = extract_mm_urls(messages) - assert result == {"audio_url": [{"Url": "data:audio/wav;base64,UklGRg=="}]} + assert result == ( + {"audio_url": [{"Url": "data:audio/wav;base64,UklGRg=="}]}, + None, + ) def test_extracts_video_urls(): @@ -70,7 +73,10 @@ def test_extracts_video_urls(): } ] result = extract_mm_urls(messages) - assert result == {"video_url": [{"Url": "https://example.com/clip.mp4"}]} + assert result == ( + {"video_url": [{"Url": "https://example.com/clip.mp4"}]}, + None, + ) def test_extracts_mixed_modalities(): @@ -95,11 +101,14 @@ def test_extracts_mixed_modalities(): } ] result = extract_mm_urls(messages) - assert result == { - "image_url": [{"Url": "https://example.com/img.jpg"}], - "audio_url": [{"Url": "https://example.com/audio.wav"}], - "video_url": [{"Url": "https://example.com/video.mp4"}], - } + assert result == ( + { + "image_url": [{"Url": "https://example.com/img.jpg"}], + "audio_url": [{"Url": "https://example.com/audio.wav"}], + "video_url": [{"Url": "https://example.com/video.mp4"}], + }, + None, + ) def test_extracts_multiple_items_per_modality(): @@ -120,12 +129,15 @@ def test_extracts_multiple_items_per_modality(): } ] result = extract_mm_urls(messages) - assert result == { - "image_url": [ - {"Url": "https://example.com/a.png"}, - {"Url": "https://example.com/b.png"}, - ] - } + assert result == ( + { + "image_url": [ + {"Url": "https://example.com/a.png"}, + {"Url": "https://example.com/b.png"}, + ] + }, + None, + ) def test_ignores_non_user_messages(): @@ -142,7 +154,7 @@ def test_ignores_non_user_messages(): }, {"role": "user", "content": "Hello"}, ] - assert extract_mm_urls(messages) is None + assert extract_mm_urls(messages) == (None, None) def test_handles_malformed_content_non_dict(): @@ -162,4 +174,93 @@ def test_handles_malformed_content_non_dict(): } ] result = extract_mm_urls(messages) - assert result == {"image_url": [{"Url": "https://example.com/ok.png"}]} + assert result == ({"image_url": [{"Url": "https://example.com/ok.png"}]}, None) + + +def test_extracts_url_and_uuid_only_slots_with_alignment(): + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + "uuid": "image-a", + }, + { + "type": "image_url", + "image_url": None, + "uuid": "image-b", + }, + { + "type": "image_url", + "image_url": {"url": "https://example.com/c.png"}, + }, + ], + } + ] + + assert extract_mm_urls(messages) == ( + { + "image_url": [ + {"Url": "https://example.com/a.png"}, + {"UuidOnly": "image-b"}, + {"Url": "https://example.com/c.png"}, + ] + }, + {"image_url": ["image-a", "image-b", None]}, + ) + + +def test_accepts_deprecated_nested_image_uuid() -> None: + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/a.png", + "uuid": "92b888ad-e64a-478f-b688-5091e16544e3", + }, + } + ], + } + ] + + assert extract_mm_urls(messages) == ( + {"image_url": [{"Url": "https://example.com/a.png"}]}, + {"image_url": ["92b888ad-e64a-478f-b688-5091e16544e3"]}, + ) + + +@pytest.mark.parametrize("part_type", ["audio_url", "video_url"]) +@pytest.mark.parametrize("nested", [False, True]) +def test_rejects_audio_video_cache_uuids(part_type: str, nested: bool) -> None: + media = {"url": "https://example.com/media", "uuid": "cached-media"} + part = {"type": part_type, part_type: media} + if not nested: + media.pop("uuid") + part["uuid"] = "cached-media" + + with pytest.raises(ValueError, match="supported only for image_url.*vLLM"): + extract_mm_urls([{"role": "user", "content": [part]}]) + + +@pytest.mark.parametrize( + "part", + [ + {"type": "image_url", "image_url": None}, + {"type": "image_url", "image_url": {}}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + "uuid": "", + }, + ], +) +def test_rejects_media_parts_without_valid_url_or_uuid(part): + messages = [{"role": "user", "content": [part]}] + + with pytest.raises(ValueError, match="URL or uuid|non-empty string"): + extract_mm_urls(messages) diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index 8daa97adbf23..2e959eb316e2 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -102,6 +102,26 @@ def test_defaults(self): assert sampling["repetition_penalty"] == 1.0 assert sampling["seed"] is None + def test_rejects_multimodal_cache_uuid(self): + request = { + "model": "test", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + "uuid": "cached-image", + } + ], + } + ], + } + + with pytest.raises(PreprocessError, match="supported only by the vLLM backend"): + _build_dynamo_preproc(request, [1], "test", None) + def test_top_k_zero_maps_to_negative_one(self): """SGLang uses -1 for disabled top_k, OpenAI uses 0.""" result = _build_dynamo_preproc( diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index 06db5354adff..3c84ab027e49 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -448,6 +448,60 @@ def feature(modality, mm_hash, offset, length): } +@pytest.mark.asyncio +async def test_prepare_mm_routing_opaque_uuid_skips_routing_but_keeps_transfer( + vllm_processor_module, + monkeypatch, +): + def fail_routing(*args, **kwargs): + raise AssertionError("opaque user UUIDs must not be parsed as routing hashes") + + monkeypatch.setattr( + vllm_processor_module, + "build_mm_routing_info_from_features", + fail_routing, + ) + + class Sender: + async def prepare(self, features, modality): + assert len(features) == 1 + assert modality == "image" + return {"mm_kwargs_shm": {"modality": modality}}, ["cleanup"] + + processor = vllm_processor_module.VllmProcessor.__new__( + vllm_processor_module.VllmProcessor + ) + processor.block_size = 16 + processor.nixl_mm_enabled = True + processor.use_shm_transfer = True + processor._sender = Sender() + + feature_hash = "derived-from-uuid-and-processor-kwargs" + vllm_preproc = SimpleNamespace( + prompt_token_ids=list(range(16)), + mm_features=[ + SimpleNamespace( + modality="image", + mm_hash=feature_hash, + data=object(), + mm_position=SimpleNamespace(offset=0, length=16), + ) + ], + ) + dynamo_preproc = {"multi_modal_uuids": {"image_url": ["opaque-user-key"]}} + + mm_routing_info, cleanup_items, transferred = await processor._prepare_mm_routing( + vllm_preproc, + dynamo_preproc, + ) + + assert mm_routing_info is None + assert cleanup_items == ["cleanup"] + assert transferred is True + assert dynamo_preproc["extra_args"]["mm_hashes"] == [feature_hash] + assert dynamo_preproc["extra_args"]["mm_kwargs_shm"] == {"modality": "image"} + + class TestReasoningParserMetadata: def test_no_reasoning_parser_returns_none(self): from dynamo.frontend.vllm_processor import _build_reasoning_parser_metadata @@ -524,6 +578,68 @@ def test_kv_router_copies_reasoning_metadata_to_extra_args(self): } +@pytest.mark.asyncio +async def test_build_engine_inputs_preserves_multimodal_uuids( + vllm_processor_module, +): + class Renderer: + async def process_for_engine_async(self, prompt, arrival_time): + assert prompt == { + "prompt": "rendered prompt", + "prompt_token_ids": [1, 2, 3], + "multi_modal_data": {"image": [image]}, + "multi_modal_uuids": {"image": ["opaque-user-key"]}, + "cache_salt": "salt", + "mm_processor_kwargs": {"do_sample_frames": False}, + } + assert isinstance(arrival_time, float) + return {"type": "multimodal", "mm_hashes": ["opaque-user-key"]} + + image = object() + engine_inputs = await vllm_processor_module._build_engine_inputs( + Renderer(), + { + "prompt": "rendered prompt", + "multi_modal_data": {"image": [image]}, + "multi_modal_uuids": {"image": ["opaque-user-key"]}, + }, + [1, 2, 3], + cache_salt="salt", + mm_processor_kwargs={"do_sample_frames": False}, + ) + + assert engine_inputs == { + "type": "multimodal", + "mm_hashes": ["opaque-user-key"], + } + + +def test_normalize_vllm_image_parts_lifts_deprecated_nested_uuid( + vllm_processor_module, +) -> None: + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.png", + "detail": None, + "uuid": "92b888ad-e64a-478f-b688-5091e16544e3", + }, + } + ], + } + ] + + vllm_processor_module._normalize_vllm_image_parts(messages) + + part = messages[0]["content"][0] + assert part["uuid"] == "92b888ad-e64a-478f-b688-5091e16544e3" + assert part["image_url"]["detail"] == "auto" + + class _FakeOutputProcessor: def __init__(self): self.request_states = {} diff --git a/components/src/dynamo/frontend/utils.py b/components/src/dynamo/frontend/utils.py index f248f8dffada..d49ac1bb1dab 100644 --- a/components/src/dynamo/frontend/utils.py +++ b/components/src/dynamo/frontend/utils.py @@ -99,21 +99,24 @@ def __init__(self, message: str): def extract_mm_urls( messages: list[dict[str, Any]], -) -> dict[str, list[dict[str, str]]] | None: - """Extract multimodal URLs from OpenAI chat completion messages. +) -> tuple[dict[str, list[dict[str, str]]] | None, dict[str, list[str | None]] | None,]: + """Extract media and vLLM image processor-cache UUIDs from chat messages. - Walks user message content arrays and collects ``image_url``, ``audio_url``, - and ``video_url`` entries. Returns them in the format expected by the - backend handler's ``_extract_multimodal_data()``:: + URL-backed parts become ``Url`` variants. Image parts with no URL and an + opaque ``uuid`` become ``UuidOnly`` variants for vLLM's multimodal + processor cache. The deprecated nested image UUID remains accepted, while + cache UUIDs on audio and video are rejected. Image UUID lists preserve + slot order:: - { - "image_url": [{"Url": "https://..."}, ...], - "audio_url": [{"Url": "data:audio/wav;base64,..."}], - } + ({"image_url": [{"Url": "https://..."}, {"UuidOnly": "image-1"}]}, + {"image_url": ["image-1", "image-1"]}) - Returns ``None`` if no multimodal content is found. + The UUID map is ``None`` when no user UUID is present. A media content part + with neither a URL nor UUID is rejected instead of being silently dropped. """ mm_data: dict[str, list[dict[str, str]]] = {} + mm_uuids: dict[str, list[str | None]] = {} + any_uuid = False for msg in messages: if not isinstance(msg, dict) or msg.get("role") != "user": @@ -127,14 +130,40 @@ def extract_mm_urls( part_type = part.get("type") if part_type not in _MEDIA_CONTENT_TYPES: continue + media_value = part.get(part_type) - if not isinstance(media_value, dict): - continue - url = media_value.get("url") + top_level_uuid = part.get("uuid") + nested_uuid = ( + media_value.get("uuid") if isinstance(media_value, dict) else None + ) + for uuid_value in (top_level_uuid, nested_uuid): + if uuid_value is not None and ( + not isinstance(uuid_value, str) or not uuid_value + ): + raise ValueError(f"{part_type} uuid must be a non-empty string") + + uuid_value = top_level_uuid if top_level_uuid is not None else nested_uuid + if uuid_value is not None and part_type != "image_url": + raise ValueError( + "multimodal cache UUIDs are currently supported only for " + "image_url parts with vLLM" + ) + + url = media_value.get("url") if isinstance(media_value, dict) else None if isinstance(url, str) and url: mm_data.setdefault(part_type, []).append({"Url": url}) - - return mm_data or None + elif isinstance(uuid_value, str): + mm_data.setdefault(part_type, []).append({"UuidOnly": uuid_value}) + else: + raise ValueError( + f"{part_type} part must contain a non-empty URL or uuid" + ) + + if part_type == "image_url": + mm_uuids.setdefault(part_type, []).append(uuid_value) + any_uuid |= uuid_value is not None + + return mm_data or None, mm_uuids if any_uuid else None def make_backend_error(engine_response: dict[str, Any]) -> dict[str, Any]: diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index 035a52bfddc0..d4f90ee9d054 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -190,6 +190,46 @@ def _inject_routing_metadata( target["mm_routing_info"] = mm_routing_info +async def _build_engine_inputs( + renderer: Any, + engine_prompt: dict[str, Any], + prompt_token_ids: list[int], + *, + cache_salt: str | None, + mm_processor_kwargs: dict[str, Any] | None, +) -> dict[str, Any]: + """Convert a rendered chat prompt into the EngineInput vLLM expects.""" + prompt_inputs = {**engine_prompt, "prompt_token_ids": prompt_token_ids} + if cache_salt is not None: + prompt_inputs["cache_salt"] = cache_salt + if mm_processor_kwargs is not None: + prompt_inputs["mm_processor_kwargs"] = mm_processor_kwargs + + return await renderer.process_for_engine_async(prompt_inputs, time.time()) + + +def _normalize_vllm_image_parts(messages: list[Any]) -> None: + """Normalize image parts before vLLM validates and renders them.""" + for msg in messages: + if not isinstance(msg, dict): + continue + content = msg.get("content") + if not isinstance(content, list): + continue + for part in content: + if not isinstance(part, dict) or part.get("type") != "image_url": + continue + image_url = part.get("image_url") + if not isinstance(image_url, dict): + continue + if image_url.get("detail") is None: + image_url["detail"] = "auto" + # Older Dynamo clients nested a UUID inside image_url. Lift it into + # vLLM's canonical content-part field before the vLLM parser sees it. + if part.get("uuid") is None and image_url.get("uuid") is not None: + part["uuid"] = image_url["uuid"] + + class VllmProcessor: def __init__( self, @@ -261,10 +301,15 @@ async def _prepare_mm_routing( rng_routing = _nvtx.start_range("mm_frontend:build_routing_info", color="cyan") if vllm_preproc.mm_features: - mm_routing_info = build_mm_routing_info_from_features( - vllm_preproc.mm_features, - prompt_token_ids=list(vllm_preproc.prompt_token_ids), - ) + # A user-provided UUID is an opaque cache identity, not necessarily a + # hexadecimal content hash. Keep these requests on text-prefix routing; + # the feature metadata is still forwarded for backend cache reuse and + # optional tensor transfer. + if not dynamo_preproc.get("multi_modal_uuids"): + mm_routing_info = build_mm_routing_info_from_features( + vllm_preproc.mm_features, + prompt_token_ids=list(vllm_preproc.prompt_token_ids), + ) ( mm_hashes_list, mm_placeholders_list, @@ -386,22 +431,12 @@ async def _generator_inner( ) -> AsyncGenerator[dict[str, Any], None]: request_id = random_uuid() - # vLLM's Pydantic model requires image_url.detail to be 'auto'/'low'/'high'. - # The Rust HTTP layer accepts None/missing, so normalize before validation. messages = request.get("messages") or [] - for msg in messages: - if not isinstance(msg, dict): - continue - content = msg.get("content") - if not isinstance(content, list): - continue - for part in content: - if not isinstance(part, dict): - continue - if part.get("type") == "image_url": - img_url = part.get("image_url") - if isinstance(img_url, dict) and img_url.get("detail") is None: - img_url["detail"] = "auto" + _normalize_vllm_image_parts(messages) + # Validate cache-UUID modality support before vLLM downloads or + # processes media. Dynamo currently exposes vLLM cache UUIDs for + # images only. + mm_data, mm_uuids = extract_mm_urls(messages) # Images are fetched by vLLM's renderer via DynamoMediaConnector, # which wraps our ImageLoader (LRU cache + in-flight dedup). @@ -476,22 +511,20 @@ async def _generator_inner( "Logprobs requested but not supported in distributed inference mode" ) - # The renderer's process_for_engine() always returns a fully processed - # EngineInput (TokenInputs or MultiModalInputs) with a "type" key. - # Pass it directly to process_inputs() — no need to rebuild a - # TokensPrompt, and this avoids the deprecation warning. - prompt_inputs = engine_prompt - if request_for_sampling.cache_salt is not None: - prompt_inputs["cache_salt"] = request_for_sampling.cache_salt - if request_for_sampling.mm_processor_kwargs is not None: - prompt_inputs[ - "mm_processor_kwargs" - ] = request_for_sampling.mm_processor_kwargs - with _nvtx.annotate("mm_frontend:process_inputs", color="orange"): + # render_messages_async returns a raw prompt. Convert it to a typed + # EngineInput before process_inputs so vLLM preserves MM UUIDs and + # does not run its multimodal processor a second time. + engine_inputs = await _build_engine_inputs( + self.input_processor.renderer, + engine_prompt, + tokens, + cache_salt=request_for_sampling.cache_salt, + mm_processor_kwargs=request_for_sampling.mm_processor_kwargs, + ) vllm_preproc: EngineCoreRequest = self.input_processor.process_inputs( request_id, - prompt_inputs, + engine_inputs, sampling_params, GENERATION_TASKS, # vLLM 0.17.0: required supported_tasks arg ) @@ -547,6 +580,12 @@ async def _generator_inner( if reasoning_parser_kwargs is not None: dynamo_preproc["reasoning_parser_kwargs"] = reasoning_parser_kwargs + # Parse user cache identities before building routing metadata. Opaque + # UUIDs deliberately suppress multimodal exact routing while retaining + # vLLM's feature metadata for backend cache reuse and tensor transfer. + if mm_uuids: + dynamo_preproc["multi_modal_uuids"] = mm_uuids + # Extract MM routing metadata and prepare transfer. cleanup_items: list = [] try: @@ -568,7 +607,6 @@ async def _generator_inner( ) all_transferred = nixl_transferred and n_with_data == n_features if not all_transferred: - mm_data = extract_mm_urls(request.get("messages") or []) if mm_data: dynamo_preproc["multi_modal_data"] = mm_data @@ -674,7 +712,8 @@ async def _generate_and_stream( # content-part counts here too (else frontend metrics report zero media). input_tokens = len(tokens) cumulative_output_tokens = 0 - _mm_counts = extract_mm_urls(request.get("messages") or []) or {} + _mm_counts, _ = extract_mm_urls(request.get("messages") or []) + _mm_counts = _mm_counts or {} image_count = len(_mm_counts.get("image_url", [])) video_count = len(_mm_counts.get("video_url", [])) audio_count = len(_mm_counts.get("audio_url", [])) diff --git a/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py b/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py index 275a5d8bf418..4e3122a61ff0 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py +++ b/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py @@ -9,6 +9,8 @@ import logging from typing import Any, Dict, Optional +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids + logger = logging.getLogger(__name__) IMAGE_URL_KEY = "image_url" @@ -54,6 +56,9 @@ def _raw_multimodal_content_types(request: Dict[str, Any]) -> set[str]: def raise_if_unextracted_multimodal(request: Dict[str, Any]) -> None: """Reject raw multimodal messages that were not extracted by the frontend.""" + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="SGLang" + ) mm_data = _multi_modal_data(request) raw_types = _raw_multimodal_content_types(request) if not (mm_data or raw_types): diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py index cbe86a8aac10..a9a02479e0a6 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py @@ -25,6 +25,7 @@ MultimodalEmbeddingCacheManager, ) from dynamo.common.multimodal import EMBEDDING_SENDER_FACTORIES +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.common.utils import nvtx_utils as _nvtx from dynamo.llm import MultimodalEmbeddingCachePublisher from dynamo.sglang.args import Config @@ -435,6 +436,9 @@ def _extract_media_urls( Returns: Tuple of (image_urls, video_urls) lists. """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="SGLang" + ) mm_data = request.get("multi_modal_data") if not mm_data: raise ValueError("multi_modal_data is required for the encode worker.") diff --git a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py index 74193a180d13..4faa124d53de 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py +++ b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py @@ -354,6 +354,15 @@ def test_raises_for_audio_url(self): def test_text_only_request_bypasses_guard(self): raise_if_unextracted_multimodal({"token_ids": [10, 20, 30]}) + def test_rejects_multimodal_cache_uuid(self): + with pytest.raises(ValueError, match="supported only by the vLLM backend"): + raise_if_unextracted_multimodal( + { + "token_ids": [10, 20, 30], + "multi_modal_uuids": {"image_url": ["cached-image"]}, + } + ) + def test_build_logprob_kwargs_allows_chosen_token_logprobs(monkeypatch): monkeypatch.delenv("DYN_SGL_ALLOW_TOP_LOGPROBS", raising=False) diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index e8393824ca80..42d23f0f6736 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -65,6 +65,18 @@ def test_extract_media_urls_supports_mixed_image_and_video(): assert video_urls == ["https://example.com/clip.mp4"] +def test_extract_media_urls_rejects_multimodal_cache_uuid(): + handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler) + + with pytest.raises(ValueError, match="supported only by the vLLM backend"): + handler._extract_media_urls( + { + "multi_modal_data": {"image_url": [{"UuidOnly": "cached-image"}]}, + "multi_modal_uuids": {"image_url": ["cached-image"]}, + } + ) + + @pytest.mark.asyncio async def test_build_mm_items_routes_video_to_video_data(): embeddings = torch.arange(24, dtype=torch.float16).reshape(6, 4) diff --git a/components/src/dynamo/trtllm/request_handlers/aggregated_handler.py b/components/src/dynamo/trtllm/request_handlers/aggregated_handler.py index 2e66fdb47070..30d39bd4367c 100644 --- a/components/src/dynamo/trtllm/request_handlers/aggregated_handler.py +++ b/components/src/dynamo/trtllm/request_handlers/aggregated_handler.py @@ -13,6 +13,7 @@ from dynamo.common.memory.multimodal_embedding_cache_manager import ( MultimodalEmbeddingCacheManager, ) +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.trtllm.multimodal.embedding_fetcher import fetch_embeddings_from_encoder from dynamo.trtllm.request_handlers.handler_base import ( HandlerBase, @@ -40,6 +41,9 @@ async def generate( self, request: dict, context: Context ) -> AsyncGenerator[dict, None]: """Generate response, optionally using remote encoder for multimodal.""" + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) logging.debug(f"AggregatedHandler Request ID: {context.id()}") embeddings: Optional[Union[torch.Tensor, dict]] = None diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index 5996eb6c46d9..cdde23ba2be1 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -37,6 +37,7 @@ from dynamo.common.backend import logprobs as _shared_logprobs from dynamo.common.backend.engine import is_generation_stage from dynamo.common.constants import DisaggregationMode as CommonDisaggregationMode +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.common.utils.structural_tag import serialize_structural_tag from dynamo.health_check import HEALTH_CHECK_KEY from dynamo.llm.exceptions import EngineShutdown @@ -789,6 +790,10 @@ async def _prepare_input_for_generation( Returns: Processed input for TRT-LLM (dict with prompt/token_ids, or raw token_ids) """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) + # DECODE mode: Use prefill metadata to skip re-processing multimodal content # Per TRT-LLM team: DECODE never needs to reload images - KV cache has the context has_prefill_metadata = epd_metadata and ( diff --git a/components/src/dynamo/trtllm/request_handlers/handlers.py b/components/src/dynamo/trtllm/request_handlers/handlers.py index d9c9a247ba4c..247cc453ce93 100644 --- a/components/src/dynamo/trtllm/request_handlers/handlers.py +++ b/components/src/dynamo/trtllm/request_handlers/handlers.py @@ -9,6 +9,7 @@ from dynamo.common.memory.multimodal_embedding_cache_manager import ( MultimodalEmbeddingCacheManager, ) +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.runtime.logging import configure_dynamo_logging from dynamo.trtllm.encode_helper import EncodeHelper from dynamo.trtllm.multimodal.embedding_fetcher import fetch_embeddings_from_encoder @@ -69,6 +70,9 @@ def __init__(self, config: RequestHandlerConfig): async def generate( self, request: dict, context: Context ) -> AsyncGenerator[dict, None]: + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) logging.debug(f"New Request ID: {context.id()}") if self.multimodal_processor is None: logging.error("encode handler: no multimodal_processor configured") @@ -136,6 +140,9 @@ async def generate( Prefill worker: process prompt and return disaggregated_params. Frontend routes to decode workers automatically. """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) logging.debug(f"Prefill Request ID: {context.id()}") request_token_ids = request.get("token_ids") logging.debug( @@ -220,6 +227,9 @@ async def generate( Decode worker: generate tokens using disaggregated_params from prefill. If disaggregated_params is present, prefill was done. Otherwise generate normally. """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) logging.debug(f"Decode Request ID: {context.id()}") disaggregated_params = request.get("disaggregated_params") diff --git a/components/src/dynamo/trtllm/tests/request_handlers/test_trtllm_uuid_guard.py b/components/src/dynamo/trtllm/tests/request_handlers/test_trtllm_uuid_guard.py new file mode 100644 index 000000000000..be4fb2a5c6dc --- /dev/null +++ b/components/src/dynamo/trtllm/tests/request_handlers/test_trtllm_uuid_guard.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for rejecting vLLM-only multimodal cache UUIDs.""" + +import pytest +import torch + +if not torch.cuda.is_available(): + pytest.skip( + "Skipping because TensorRT-LLM imports require a CUDA-capable GPU.", + allow_module_level=True, + ) + +from dynamo.trtllm.request_handlers.aggregated_handler import AggregatedHandler +from dynamo.trtllm.request_handlers.handler_base import HandlerBase +from dynamo.trtllm.request_handlers.handlers import ( + DecodeHandler, + EncodeHandler, + PrefillHandler, +) +from dynamo.trtllm.tests.request_handlers.utils import create_mock_context +from dynamo.trtllm.tests.utils import create_mock_request_handler_config + +pytestmark = [ + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.trtllm, + pytest.mark.gpu_0, +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("handler_type", "disaggregation_mode"), + [ + (EncodeHandler, "encode"), + (PrefillHandler, "prefill"), + (DecodeHandler, "decode"), + (AggregatedHandler, "prefill_and_decode"), + ], +) +async def test_handler_rejects_multimodal_cache_uuid( + handler_type: type[HandlerBase], disaggregation_mode: str +) -> None: + config = create_mock_request_handler_config(disaggregation_mode=disaggregation_mode) + handler = handler_type(config) + request = {"multi_modal_uuids": {"image_url": ["cached-image"]}} + + with pytest.raises(ValueError, match="supported only by the vLLM backend"): + async for _ in handler.generate(request, create_mock_context()): + pass diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index 030d5a0d0dcd..8cc5578ae330 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -625,6 +625,18 @@ async def test_text_only_request_falls_back_to_token_ids(self): result = await self._prepare(handler, {"token_ids": [10, 20, 30]}) assert result == [10, 20, 30] + @pytest.mark.asyncio + async def test_rejects_multimodal_cache_uuid(self): + handler = self._make_handler(multimodal_processor=MagicMock()) + request = { + "token_ids": [1, 2, 3], + "multi_modal_data": {"image_url": [{"UuidOnly": "cached-image"}]}, + "multi_modal_uuids": {"image_url": ["cached-image"]}, + } + + with pytest.raises(ValueError, match="supported only by the vLLM backend"): + await self._prepare(handler, request) + @pytest.mark.asyncio async def test_decode_with_prefill_metadata_bypasses_guard(self): handler = self._make_handler(multimodal_processor=None) diff --git a/components/src/dynamo/vllm/multimodal_utils/multimodal_embedding_cache_connector.py b/components/src/dynamo/vllm/multimodal_utils/multimodal_embedding_cache_connector.py index d91f6a917367..afcdf0ed331d 100644 --- a/components/src/dynamo/vllm/multimodal_utils/multimodal_embedding_cache_connector.py +++ b/components/src/dynamo/vllm/multimodal_utils/multimodal_embedding_cache_connector.py @@ -23,7 +23,10 @@ MINIMUM_VLLM_VERSION = "0.17.0" -logger = logging.getLogger(__name__) +# This connector is instantiated inside vLLM's spawned EngineCore process. +# That process configures the ``vllm`` logger hierarchy, but not Dynamo's root +# logger, so use a vLLM child logger to keep connector diagnostics observable. +logger = logging.getLogger("vllm.dynamo.multimodal_embedding_cache_connector") def _get_device(vllm_config: "VllmConfig") -> str: @@ -145,6 +148,9 @@ def has_cache_item(self, identifier: str) -> bool: """ if identifier in self._cache_order: self._cache_order.move_to_end(identifier) + logger.debug( + "Dynamo multimodal embedding cache hit: identifier=%r", identifier + ) return True return False diff --git a/components/src/dynamo/vllm/multimodal_utils/request_processor.py b/components/src/dynamo/vllm/multimodal_utils/request_processor.py index 8bf8bacd7420..205d13dd71cc 100644 --- a/components/src/dynamo/vllm/multimodal_utils/request_processor.py +++ b/components/src/dynamo/vllm/multimodal_utils/request_processor.py @@ -13,6 +13,7 @@ import logging import pickle +from collections.abc import Sequence from dataclasses import dataclass from typing import Any, Optional @@ -49,7 +50,9 @@ URL_VARIANT_KEY = "Url" -def pad_mm_hashes_to_64(mm_hashes: list[str]) -> list[str]: +def pad_mm_hashes_to_64( + mm_hashes: Sequence[str | None], +) -> list[str | None]: """Pad frontend hashes to vLLM's 64-character UUID representation.""" return [ value.ljust(64, "0") if isinstance(value, str) and len(value) < 64 else value @@ -70,11 +73,11 @@ def _normalize_forwarded_mm_modality( def _build_forwarded_mm_uuids( extra_args: dict[str, Any], use_unified_vision_chunk: bool, -) -> Optional[dict[str, Any]]: +) -> Optional[dict[str, list[str | None]]]: """Preserve frontend cache identities, including mixed modalities.""" grouped_hashes = extra_args.get("mm_hashes_by_modality") if isinstance(grouped_hashes, dict): - mm_uuids: dict[str, Any] = {} + mm_uuids: dict[str, list[str | None]] = {} for modality, hashes in grouped_hashes.items(): if not hashes: continue @@ -94,11 +97,43 @@ def _build_forwarded_mm_uuids( "image", use_unified_vision_chunk, ) - return {modality_key: pad_mm_hashes_to_64(list(forwarded_hashes))} + padded_hashes = pad_mm_hashes_to_64(list(forwarded_hashes)) + return {modality_key: padded_hashes} return None +def _build_user_mm_uuids( + raw_uuids: Any, + use_unified_vision_chunk: bool, +) -> Optional[dict[str, list[str | None]]]: + """Normalize vLLM image cache identities without changing opaque values.""" + if not isinstance(raw_uuids, dict): + return None + + for modality, values in raw_uuids.items(): + if modality == IMAGE_URL_KEY: + continue + has_uuid = ( + any(value is not None for value in values) + if isinstance(values, list) + else values is not None + ) + if has_uuid: + raise ValueError( + "multimodal cache UUIDs are currently supported only for images" + ) + + image_uuids = raw_uuids.get(IMAGE_URL_KEY) + if not isinstance(image_uuids, list): + return None + backend_modality = _normalize_forwarded_mm_modality( + "image", + use_unified_vision_chunk, + ) + return {backend_modality: list(image_uuids)} + + def _get_modality_extra_values( extra_args: dict[str, Any], grouped_key: str, @@ -142,7 +177,7 @@ def _placeholder_range_from_extra_arg(value: Any) -> PlaceholderRange: def compute_mm_uuids( multi_modal_data: Optional[dict[str, Any]], -) -> Optional[dict[str, list[str]]]: +) -> Optional[dict[str, list[str | None]]]: """Compute image UUIDs when the frontend did not provide canonical hashes.""" if not multi_modal_data: return None @@ -170,7 +205,8 @@ def compute_mm_uuids( images = [images] if not images: return None - return {modality: compute_mm_uuids_from_images(images)} + uuids: list[str | None] = list(compute_mm_uuids_from_images(images)) + return {modality: uuids} def get_mm_processor_kwargs(request: dict[str, Any]) -> Optional[dict[str, Any]]: @@ -270,7 +306,9 @@ def validate_multimodal_request(self, request: dict[str, Any]) -> None: for key in ("mm_kwargs_shm", "mm_kwargs_nixl") ) if ( - request.get("multi_modal_data") is not None or has_transfer + request.get("multi_modal_data") is not None + or request.get("multi_modal_uuids") is not None + or has_transfer ) and not self.enable_multimodal: raise self._multimodal_disabled_error() @@ -331,7 +369,7 @@ async def extract_multimodal_data( for item in mm_map.get(IMAGE_URL_KEY, []): if isinstance(item, dict) and URL_VARIANT_KEY in item: image_urls.append(item[URL_VARIANT_KEY]) - elif isinstance(item, dict) and "Decoded" in item: + else: supported = False if supported: vllm_mm_data = ( @@ -359,7 +397,9 @@ async def extract_multimodal_data( ) else: vllm_mm_data[image_key] = ( - images[0] if len(images) == 1 else images + images[0] + if len(images) == 1 and images[0] is not None + else images ) video_items = mm_map.get(VIDEO_URL_KEY, []) @@ -426,7 +466,10 @@ async def try_receive_mm_kwargs( if shm_meta_raw: shm_metadata = MmKwargsShmTransferMetadata.model_validate(shm_meta_raw) return await self._receive_mm_kwargs( - extra_args, "shm", MmKwargsShmReceiver(), shm_metadata + extra_args, + "shm", + MmKwargsShmReceiver(), + shm_metadata, ) if nixl_meta_raw: @@ -434,7 +477,10 @@ async def try_receive_mm_kwargs( if self._mm_kwargs_receiver is None: self._mm_kwargs_receiver = MmKwargsNixlReceiver() return await self._receive_mm_kwargs( - extra_args, "nixl", self._mm_kwargs_receiver, nixl_metadata + extra_args, + "nixl", + self._mm_kwargs_receiver, + nixl_metadata, ) except Exception: logger.exception( @@ -509,8 +555,12 @@ async def _receive_mm_kwargs( ) return None - padded_hashes = pad_mm_hashes_to_64(list(mm_hashes)) - mm_hashes_dict = {backend_modality: padded_hashes} + # These are vLLM's final feature hashes. When the request supplies + # an opaque UUID, vLLM derives this identity from the UUID together + # with mm_processor_kwargs. Any rewriting (including zero-padding) + # would create a different worker-cache key. + feature_hashes = list(mm_hashes) + mm_hashes_dict = {backend_modality: feature_hashes} mm_kwargs_dict = {backend_modality: kwargs_items} engine_input = { "type": "multimodal", @@ -549,10 +599,15 @@ def build_tokens_prompt( ) -> TokensPrompt: """Create a TokensPrompt with stable multimodal UUIDs.""" extra_args = request.get("extra_args") or {} - mm_uuids = _build_forwarded_mm_uuids( - extra_args, + mm_uuids = _build_user_mm_uuids( + request.get("multi_modal_uuids"), self.use_unified_vision_chunk, ) + if mm_uuids is None: + mm_uuids = _build_forwarded_mm_uuids( + extra_args, + self.use_unified_vision_chunk, + ) if mm_uuids is None and self.embedding_loader is None: mm_uuids = compute_mm_uuids(multi_modal_data) if mm_uuids is not None: diff --git a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_multimodal_embedding_cache_connector.py b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_multimodal_embedding_cache_connector.py index cd06a84d8b00..dab5e362ee14 100644 --- a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_multimodal_embedding_cache_connector.py +++ b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_multimodal_embedding_cache_connector.py @@ -128,12 +128,18 @@ def get_num_encoder_embeds(idx): def test_has_cache_item_miss_then_hit(self): conn = self._make_connector() - assert not conn.has_cache_item("hash_a") + opaque_uuid = "catalog/image:v2" + assert not conn.has_cache_item(opaque_uuid) - request = self._make_request([("hash_a", 100)]) + request = self._make_request([(opaque_uuid, 100)]) conn.update_state_after_alloc(request, 0) - assert conn.has_cache_item("hash_a") + with patch.object(mod.logger, "debug") as log_debug: + assert conn.has_cache_item(opaque_uuid) + log_debug.assert_called_once_with( + "Dynamo multimodal embedding cache hit: identifier=%r", + opaque_uuid, + ) def test_update_state_plans_save(self): conn = self._make_connector() diff --git a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py index 635ef83f635b..4eecfeb33735 100644 --- a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py +++ b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py @@ -114,6 +114,26 @@ async def test_merges_encoder_images_with_local_video_and_decoded_fallback(): processor.embedding_loader.load_multimodal_embeddings.assert_awaited_once() +@pytest.mark.asyncio +async def test_extracts_uuid_only_media_as_aligned_none_slots(): + processor = _processor() + image = Image.new("RGB", (1, 1)) + image_items = [ + {"Url": "https://example.com/image.png"}, + {"UuidOnly": "cached-image"}, + ] + processor.image_loader.load_image_batch.return_value = [image, None] + + result = await processor.extract_multimodal_data( + {"multi_modal_data": {"image_url": image_items}}, + "request-cached-image", + None, + ) + + assert result == {"image": [image, None]} + processor.image_loader.load_image_batch.assert_awaited_once_with(image_items) + + @pytest.mark.asyncio async def test_rejects_media_when_multimodal_is_disabled(): processor = _processor(enabled=False) @@ -286,6 +306,50 @@ def test_build_tokens_prompt_forwards_hashes_kwargs_and_vision_chunk(): assert prompt["mm_processor_kwargs"] == {"num_crops": 4} +def test_build_tokens_prompt_prefers_opaque_user_uuids_without_padding(): + processor = _processor() + mm_data = {"image": [object(), None]} + + prompt = processor.build_tokens_prompt( + { + "token_ids": [1, 2, 3], + "multi_modal_uuids": {"image_url": ["sku-image-a", "sku-image-b"]}, + "extra_args": {"mm_hashes": ["routing-a", "routing-b"]}, + }, + mm_data, + None, + ) + + assert prompt["multi_modal_data"] is mm_data + assert prompt["multi_modal_uuids"] == {"image": ["sku-image-a", "sku-image-b"]} + + +@pytest.mark.parametrize( + "unsupported_uuids", + [ + {"video_url": ["video-key"]}, + {"audio_url": "audio-key"}, + ], +) +def test_build_tokens_prompt_rejects_user_audio_video_uuids( + unsupported_uuids: dict[str, object], +) -> None: + processor = _processor(unified_vision_chunk=True) + + with pytest.raises(ValueError, match="supported only for images"): + processor.build_tokens_prompt( + { + "token_ids": [1, 2, 3], + "multi_modal_uuids": { + "image_url": ["image-key"], + **unsupported_uuids, + }, + }, + {"vision_chunk": [None], "video": [None], "audio": [None]}, + None, + ) + + def test_build_tokens_prompt_preserves_grouped_forwarded_hashes(): processor = _processor() @@ -392,6 +456,7 @@ def test_forwarded_placeholder_preserves_is_embed_mask(): ([], []), (["0123456789abcdef"], ["0123456789abcdef" + "0" * 48]), (["f" * 64], ["f" * 64]), + (["opaque-key", None], ["opaque-key" + "0" * 54, None]), ], ) def test_pad_mm_hashes_to_64(hashes, expected): @@ -606,12 +671,42 @@ async def test_receive_transferred_kwargs_injects_vllm_cache(monkeypatch): metadata, ) - padded_hash = "hash".ljust(64, "0") assert result is not None assert result["prompt_token_ids"] == [10, 11, 12] - assert result["mm_hashes"] == {"image": [padded_hash]} + assert result["mm_hashes"] == {"image": ["hash"]} + input_processor.inject_into_mm_cache.assert_called_once_with( + {"image": ["hash"]}, {"image": [item]} + ) + + +@pytest.mark.asyncio +async def test_receive_transferred_kwargs_keeps_vllm_feature_hash(monkeypatch): + input_processor = SimpleNamespace(inject_into_mm_cache=MagicMock()) + processor = _processor() + processor.engine_client = SimpleNamespace(input_processor=input_processor) + item = MagicMock(spec=mod.MultiModalKwargsItem) + monkeypatch.setattr(mod.pickle, "loads", lambda payload: item) + receiver = SimpleNamespace( + receive=AsyncMock(return_value={"__pickled_kwargs_item__": [b"payload"]}) + ) + + result = await processor._receive_mm_kwargs( + { + # vLLM derives this hash from the opaque user UUID together with + # mm_processor_kwargs; the raw UUID must not replace it. + "mm_hashes": ["derived-feature-hash"], + "mm_placeholders": [[1, 2]], + "expanded_token_ids": [10, 11, 12], + }, + "shm", + receiver, + SimpleNamespace(modality="image", mm_hashes=[]), + ) + + assert result is not None + assert result["mm_hashes"] == {"image": ["derived-feature-hash"]} input_processor.inject_into_mm_cache.assert_called_once_with( - {"image": [padded_hash]}, {"image": [item]} + {"image": ["derived-feature-hash"]}, {"image": [item]} ) @@ -648,13 +743,12 @@ async def test_receive_transferred_kwargs_uses_grouped_metadata_and_vision_chunk metadata, ) - padded_hash = "grouped_hash".ljust(64, "0") assert result is not None - assert result["mm_hashes"] == {"vision_chunk": [padded_hash]} + assert result["mm_hashes"] == {"vision_chunk": ["grouped_hash"]} placeholder = result["mm_placeholders"]["vision_chunk"][0] assert placeholder.get_num_embeds() == 1 input_processor.inject_into_mm_cache.assert_called_once_with( - {"vision_chunk": [padded_hash]}, {"vision_chunk": [item]} + {"vision_chunk": ["grouped_hash"]}, {"vision_chunk": [item]} ) @@ -678,7 +772,7 @@ async def test_receive_transferred_kwargs_falls_back_to_metadata_hashes(monkeypa ) assert result is not None - assert result["mm_hashes"] == {"video": ["metadata_hash".ljust(64, "0")]} + assert result["mm_hashes"] == {"video": ["metadata_hash"]} @pytest.mark.asyncio diff --git a/docs/features/multimodal/multimodal-vllm.md b/docs/features/multimodal/multimodal-vllm.md index ee4c158196ea..5d6efd18858c 100644 --- a/docs/features/multimodal/multimodal-vllm.md +++ b/docs/features/multimodal/multimodal-vllm.md @@ -102,6 +102,69 @@ curl http://localhost:8000/v1/chat/completions \ }' ``` +### Reuse vLLM Multimodal Processor Cache Entries + +vLLM can cache processed multimodal inputs under a client-provided opaque UUID. +Dynamo currently exposes this behavior for images only. The extension is +specific to vLLM; it is not part of the OpenAI Chat Completions API and is not +supported by Dynamo's other backends. Dynamo rejects UUIDs on audio or video, +and its SGLang and TensorRT-LLM backends reject image UUIDs rather than silently +ignoring unsupported cache semantics. + +To enable the cache, pass a nonzero `--mm-processor-cache-gb` value to the vLLM +worker. + +For an aggregated worker, enable Dynamo's CPU embedding cache alongside the +vLLM processor cache: + +```bash +bash launch/agg_multimodal.sh --model Qwen/Qwen3-VL-2B-Instruct \ + --mm-processor-cache-gb 4 \ + --multimodal-embedding-cache-capacity-gb 1 +``` + +The caches store different data. The vLLM processor cache retains the processed +media metadata required to resolve a UUID-only request. Dynamo's embedding cache +retains encoder output and can restore it after eviction from vLLM's GPU encoder +cache. Keep both caches enabled; the embedding cache alone cannot reconstruct a +UUID-only input. Both caches are local to one vLLM engine, so the fill and reuse +requests must reach the same aggregated worker. + +Populate an entry by adding `uuid` beside the media field: + +```json +{ + "type": "image_url", + "image_url": { + "url": "https://example.com/image.png" + }, + "uuid": "catalog-image-42" +} +``` + +Reuse that entry in a later request by setting the media field to `null` and +sending the same top-level UUID: + +```json +{ + "type": "image_url", + "image_url": null, + "uuid": "catalog-image-42" +} +``` + +UUIDs are opaque nonempty strings. The top-level field is canonical. Dynamo +still accepts the deprecated nested `image_url.uuid` form for compatibility, +but nested values must be UUID-formatted and cannot express UUID-only requests. +A UUID-only request fails on a cache miss because it contains no media payload +to process. Dynamo also rejects a media content part that has neither a URL nor +a UUID. + +When KV-aware routing is enabled, requests with client UUIDs use text-prefix +routing. Dynamo cannot convert an opaque client key into its content-derived +multimodal routing hash without risking different cache identities at the +router and worker. + **Video request:** ```bash diff --git a/examples/backends/trtllm/mm_router_worker/handler.py b/examples/backends/trtllm/mm_router_worker/handler.py index 078d223c91c0..2ff07b1c8a69 100644 --- a/examples/backends/trtllm/mm_router_worker/handler.py +++ b/examples/backends/trtllm/mm_router_worker/handler.py @@ -8,6 +8,7 @@ import logging from typing import Any, AsyncGenerator +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.llm import KvRouter from dynamo.runtime.logging import configure_dynamo_logging @@ -68,6 +69,10 @@ async def generate(self, request: dict) -> AsyncGenerator[dict, None]: Yields: Response chunks from the downstream TRT-LLM worker via KvRouter """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) + # Extract messages from extra_args (set by Frontend preprocessor) messages = request.get("extra_args", {}).get("messages", []) image_urls = extract_image_urls(messages) diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index 828a29630de8..4d5ea47221aa 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -1713,9 +1713,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18f8f6a2d325675a65beb0718940124a011e5ea25ee827df19028b965296a6c" +checksum = "bf45340e7f5beb8f3055c1fb933394f05256b6548a1b1434ae3df27bfc8d2f6e" dependencies = [ "aho-corasick", "anyhow", @@ -1751,9 +1751,9 @@ dependencies = [ [[package]] name = "dynamo-protocols" -version = "2.0.2" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1da866e573424e374c83d64dbf7807547054ebadcd85ddb2f9f67db06bf386" +checksum = "aaf9875c31601458f29b83e5ed0bf63479dc3a086e4131bbee0863101f4fd5f3" dependencies = [ "async-openai", "derive_builder", @@ -1768,9 +1768,9 @@ dependencies = [ [[package]] name = "dynamo-renderer" -version = "1.3.8" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3472f4ee978f581bac985c4f5ebad55a9b1b4ccb025ed3e9df708efa56e83bfa" +checksum = "1afd1dc880ea81af098f1d0586af1ab37865c82f87d14265b7e72bf00696ffef" dependencies = [ "anyhow", "chrono", @@ -1867,9 +1867,9 @@ dependencies = [ [[package]] name = "dynamo-tokenizers" -version = "1.5.0" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb9c031ea19d99d1faf8ff304fe666eb2dbd4666fcb416af3f08ae88f5be80" +checksum = "2843cd9cd50002ae23f3df525e1c551775b7bc7c9041742539faf63bc35ea225" dependencies = [ "aho-corasick", "anyhow", diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index ef1707fa0efa..87789a0a097e 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2326,9 +2326,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18f8f6a2d325675a65beb0718940124a011e5ea25ee827df19028b965296a6c" +checksum = "bf45340e7f5beb8f3055c1fb933394f05256b6548a1b1434ae3df27bfc8d2f6e" dependencies = [ "aho-corasick", "anyhow", @@ -2364,9 +2364,9 @@ dependencies = [ [[package]] name = "dynamo-protocols" -version = "2.0.2" +version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1da866e573424e374c83d64dbf7807547054ebadcd85ddb2f9f67db06bf386" +checksum = "aaf9875c31601458f29b83e5ed0bf63479dc3a086e4131bbee0863101f4fd5f3" dependencies = [ "async-openai", "derive_builder", @@ -2421,9 +2421,9 @@ dependencies = [ [[package]] name = "dynamo-renderer" -version = "1.3.8" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3472f4ee978f581bac985c4f5ebad55a9b1b4ccb025ed3e9df708efa56e83bfa" +checksum = "1afd1dc880ea81af098f1d0586af1ab37865c82f87d14265b7e72bf00696ffef" dependencies = [ "anyhow", "chrono", @@ -2543,9 +2543,9 @@ dependencies = [ [[package]] name = "dynamo-tokenizers" -version = "1.5.0" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb9c031ea19d99d1faf8ff304fe666eb2dbd4666fcb416af3f08ae88f5be80" +checksum = "2843cd9cd50002ae23f3df525e1c551775b7bc7c9041742539faf63bc35ea225" dependencies = [ "aho-corasick", "anyhow", diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index 1dbb8075ee51..e33281b43017 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -49,7 +49,7 @@ mm-routing = ["dynamo-llm/mm-routing"] aiconfigurator-core = { git = "https://github.com/ai-dynamo/aiconfigurator.git", rev = "ffbab15797d3a3f14c382e937c0ab0c1c5cbc530", package = "aiconfigurator-core", optional = true } dynamo-runtime = { path = "../../runtime" } dynamo-mocker = { path = "../../mocker" } -dynamo-parsers = { version = "5.0.0" } +dynamo-parsers = { version = "5.1.0" } dynamo-backend-common = { path = "../../backend-common" } dynamo-sglang-sidecar = { path = "../../backend/sglang-sidecar" } diff --git a/lib/llm/src/entrypoint/input/common.rs b/lib/llm/src/entrypoint/input/common.rs index af75cb532729..29b5ae7be645 100644 --- a/lib/llm/src/entrypoint/input/common.rs +++ b/lib/llm/src/entrypoint/input/common.rs @@ -68,7 +68,7 @@ fn preprocessed_multimodal_cache_keys(request: &PreprocessedRequest) -> Vec keys.push(multimodal_cache_key_from_url(url.as_str())), MultimodalData::RawUrl(url) => keys.push(multimodal_cache_key_from_url(url)), - MultimodalData::Decoded(_) => {} + MultimodalData::Decoded(_) | MultimodalData::UuidOnly(_) => {} } } keys.sort(); diff --git a/lib/llm/src/model_card.rs b/lib/llm/src/model_card.rs index eb62b176ccef..d20c0e61a9c1 100644 --- a/lib/llm/src/model_card.rs +++ b/lib/llm/src/model_card.rs @@ -61,9 +61,12 @@ fn instrumented_tokenizer_cache( cache_bytes: usize, cache_extend: bool, model: &str, -) -> Arc { - Arc::new( - crate::tokenizers::CachedTokenizer::new(raw, special_tokens, cache_bytes) +) -> Result> { + let cached = crate::tokenizers::CachedTokenizer::new(raw, special_tokens, cache_bytes) + .context("failed to initialize tokenizer prefix cache")?; + + Ok(Arc::new( + cached .with_extend(cache_extend) .with_observer( Arc::new(|| { @@ -74,7 +77,7 @@ fn instrumented_tokenizer_cache( }), ) .with_token_observer(tokenizer_cache_token_observer(model)), - ) + )) } /// Identify model deployment cards in the key-value store @@ -1240,7 +1243,7 @@ impl ModelDeploymentCard { cache_bytes, cache_extend, self.name(), - ) + )? } else { raw } @@ -1272,7 +1275,7 @@ impl ModelDeploymentCard { cache_bytes, cache_extend, self.name(), - ) + )? } else { raw } diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 5f9e35e6e112..47cb0f4c5f02 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -47,7 +47,7 @@ use crate::model_card::ModelInfoType; use crate::model_card::{ModelDeploymentCard, ModelInfo}; use crate::preprocessor::media::MediaLoader; use crate::protocols::common::preprocessor::{ - MultimodalData, MultimodalDataMap, PreprocessedRequestBuilder, RoutingHints, + MultimodalData, MultimodalDataMap, MultimodalUuidMap, PreprocessedRequestBuilder, RoutingHints, }; use crate::protocols::common::timing::RequestTracker; use crate::tokenizers::Encoding; @@ -1244,6 +1244,7 @@ impl OpenAIPreprocessor { } } + #[allow(deprecated)] pub async fn gather_multi_modal_data< R: OAIChatLikeRequest + MediaRequestExt + NvExtProvider, >( @@ -1261,8 +1262,16 @@ impl OpenAIPreprocessor { let _ = token_ids; let mut media_map: MultimodalDataMap = HashMap::new(); - let mut fetch_tasks: Vec<(String, &ChatCompletionRequestUserMessageContentPart)> = - Vec::new(); + let mut uuid_map: MultimodalUuidMap = HashMap::new(); + let mut any_user_uuid = false; + // Decoded results are written back into these reserved modality slots so + // URL-backed and UUID-only inputs retain request order. + let mut fetch_tasks: Vec<( + String, + usize, + String, + &ChatCompletionRequestUserMessageContentPart, + )> = Vec::new(); // Per-image (mm_hash, width, height) for the MM-routing path. // Accumulated in message order so we don't walk messages twice. // Cleared and returned to the caller; empty for non-image / text-only requests. @@ -1302,41 +1311,101 @@ impl OpenAIPreprocessor { _ => continue, }; for content_part in content_parts.iter() { - if has_media_loader { - let type_str = match content_part { - ChatCompletionRequestUserMessageContentPart::ImageUrl(_) => "image_url", - ChatCompletionRequestUserMessageContentPart::VideoUrl(_) => "video_url", - ChatCompletionRequestUserMessageContentPart::AudioUrl(_) => "audio_url", - _ => continue, - }; - #[cfg(feature = "mm-routing")] - if type_str == "image_url" { - total_image_count += 1; + let (type_str, url, uuid) = match content_part { + ChatCompletionRequestUserMessageContentPart::ImageUrl(part) => { + let legacy_uuid = part + .image_url + .as_ref() + .and_then(|media| media.uuid.as_ref()) + .map(ToString::to_string); + ( + "image_url", + part.image_url.as_ref().map(|media| media.url.clone()), + part.uuid.clone().or(legacy_uuid), + ) } - fetch_tasks.push((type_str.to_string(), content_part)); - } else { - let (type_str, url) = match content_part { - ChatCompletionRequestUserMessageContentPart::ImageUrl(p) => { - ("image_url", p.image_url.url.clone()) - } - ChatCompletionRequestUserMessageContentPart::VideoUrl(p) => { - ("video_url", p.video_url.url.clone()) + ChatCompletionRequestUserMessageContentPart::VideoUrl(part) => { + let has_legacy_uuid = part + .video_url + .as_ref() + .is_some_and(|media| media.uuid.is_some()); + if part.uuid.is_some() || has_legacy_uuid { + bail!( + "multimodal cache UUIDs are currently supported only for image_url parts with vLLM" + ); } - ChatCompletionRequestUserMessageContentPart::AudioUrl(p) => { - ("audio_url", p.audio_url.url.clone()) + ( + "video_url", + part.video_url.as_ref().map(|media| media.url.clone()), + None, + ) + } + ChatCompletionRequestUserMessageContentPart::AudioUrl(part) => { + let has_legacy_uuid = part + .audio_url + .as_ref() + .is_some_and(|media| media.uuid.is_some()); + if part.uuid.is_some() || has_legacy_uuid { + bail!( + "multimodal cache UUIDs are currently supported only for image_url parts with vLLM" + ); } - _ => continue, - }; - #[cfg(feature = "mm-routing")] - if type_str == "image_url" { - total_image_count += 1; - let mm_hash = Self::hash_image_url(url.as_str()); - url_passthrough_images.push((mm_hash, url.to_string())); + ( + "audio_url", + part.audio_url.as_ref().map(|media| media.url.clone()), + None, + ) } - media_map - .entry(type_str.to_string()) + _ => continue, + }; + + #[cfg(feature = "mm-routing")] + if type_str == "image_url" { + total_image_count += 1; + } + + if uuid.as_deref().is_some_and(str::is_empty) { + bail!("{type_str} uuid must be a non-empty string"); + } + + let modality = type_str.to_string(); + let slot_idx = media_map.entry(modality.clone()).or_default().len(); + any_user_uuid |= uuid.is_some(); + if type_str == "image_url" { + uuid_map + .entry(modality.clone()) .or_default() - .push(MultimodalData::Url(url)); + .push(uuid.clone()); + } + + match (url, uuid) { + (Some(url), _) => { + media_map + .entry(modality.clone()) + .or_default() + .push(MultimodalData::Url(url.clone())); + + if has_media_loader { + fetch_tasks.push((modality, slot_idx, url.to_string(), content_part)); + } else { + #[cfg(feature = "mm-routing")] + if type_str == "image_url" { + let mm_hash = Self::hash_image_url(url.as_str()); + url_passthrough_images.push((mm_hash, url.to_string())); + } + } + } + (None, Some(uuid)) => { + media_map + .entry(modality) + .or_default() + .push(MultimodalData::UuidOnly(uuid)); + } + (None, None) => { + bail!( + "{type_str} part has neither `url` nor `uuid`; at least one is required" + ); + } } } } @@ -1345,12 +1414,15 @@ impl OpenAIPreprocessor { if !fetch_tasks.is_empty() { let loader = self.media_loader.as_ref().unwrap(); let media_io_kwargs = request.media_io_kwargs(); - let results = futures::future::join_all(fetch_tasks.iter().map(|(_, content_part)| { - loader.fetch_and_decode_media_part(content_part, media_io_kwargs) - })) - .await; + let results = + futures::future::join_all(fetch_tasks.iter().map(|(_, _, _, content_part)| { + loader.fetch_and_decode_media_part(content_part, media_io_kwargs) + })) + .await; - for ((type_str, _content_part), result) in fetch_tasks.into_iter().zip(results) { + for ((type_str, slot_idx, _source_url, _content_part), result) in + fetch_tasks.into_iter().zip(results) + { // if one item fails, errors the whole request, other items will be cleaned up by Drop let rdma_descriptor = result?; @@ -1362,14 +1434,6 @@ impl OpenAIPreprocessor { if shape.len() >= 2 { let h = shape[0] as u32; let w = shape[1] as u32; - let url_str = match _content_part { - ChatCompletionRequestUserMessageContentPart::ImageUrl(p) => { - p.image_url.url.as_str() - } - _ => unreachable!( - "rdma image_url descriptor only originates from ImageUrl content parts" - ), - }; // Frontend-decode path: hash the decoded RGB bytes so // the same image reached via different (signed) URLs // collides on the same `mm_hash` and routes to the @@ -1379,7 +1443,7 @@ impl OpenAIPreprocessor { // shouldn't happen on the frontend. let (mm_hash, hash_source) = match rdma_descriptor.content_hash() { Some(h) => (h, "decoded_bytes"), - None => (Self::hash_image_url(url_str), "url_fallback"), + None => (Self::hash_image_url(&_source_url), "url_fallback"), }; if let Some(counter) = self.image_token_counter.as_ref() { let n = counter.count_tokens(w, h); @@ -1402,10 +1466,10 @@ impl OpenAIPreprocessor { } } - media_map - .entry(type_str) - .or_default() - .push(MultimodalData::Decoded(rdma_descriptor)); + let slots = media_map + .get_mut(&type_str) + .expect("modality must exist after reserving a fetch slot"); + slots[slot_idx] = MultimodalData::Decoded(rdma_descriptor); } } @@ -1468,6 +1532,17 @@ impl OpenAIPreprocessor { if !media_map.is_empty() { builder.multi_modal_data(Some(media_map)); + if any_user_uuid { + builder.multi_modal_uuids(Some(uuid_map)); + } + + // User cache identities are opaque and cannot be converted into the + // router's canonical image hashes. Fall back to text-prefix routing + // instead of routing and publishing under different cache keys. + #[cfg(feature = "mm-routing")] + if any_user_uuid { + mm_image_entries.clear(); + } // Preserve original messages and formatted prompt in extra_args for multimodal // workers (e.g., TRT-LLM needs messages and the template-rendered prompt with @@ -1527,6 +1602,7 @@ impl OpenAIPreprocessor { // text-prefix routing. #[cfg(feature = "mm-routing")] if let Some(find_token_id) = self.routing_image_token_id + && !any_user_uuid && !mm_image_entries.is_empty() && mm_image_entries.len() == total_image_count && token_ids.iter().filter(|&&t| t == find_token_id).count() diff --git a/lib/llm/src/preprocessor/media/loader.rs b/lib/llm/src/preprocessor/media/loader.rs index 36613454f682..7696fdaf789a 100644 --- a/lib/llm/src/preprocessor/media/loader.rs +++ b/lib/llm/src/preprocessor/media/loader.rs @@ -472,8 +472,10 @@ impl MediaLoader { // output (resize, normalisation). When it's set we skip the // cache to stay correct; in practice it's None on the common // path so the hit rate is unaffected. - if media_io_kwargs.is_none() { - let key = Self::cache_key(image_part.image_url.url.as_str()); + if media_io_kwargs.is_none() + && let Some(url) = image_part.image_url.as_ref().map(|media| &media.url) + { + let key = Self::cache_key(url.as_str()); if let Some(hit) = cache.lock().get(&key) { tracing::debug!(url_hash = key, "[mm-cache] hit"); return Ok(hit); @@ -490,7 +492,15 @@ impl MediaLoader { .as_ref() .ok_or_else(|| anyhow::anyhow!("Model does not support image inputs"))?; - let url = &image_part.image_url.url; + let url = image_part + .image_url + .as_ref() + .map(|media| &media.url) + .ok_or_else(|| { + anyhow::anyhow!( + "Cannot decode an image content part without a URL; UUID-only parts must be resolved by the backend cache" + ) + })?; self.media_fetcher .check_if_url_allowed_with_dns(url) .await?; @@ -513,7 +523,13 @@ impl MediaLoader { anyhow::anyhow!("Model does not support video inputs") })?; - let url = &video_part.video_url.url; + let url = video_part + .video_url + .as_ref() + .map(|media| &media.url) + .ok_or_else(|| { + anyhow::anyhow!("Cannot decode a video content part without a URL") + })?; self.media_fetcher .check_if_url_allowed_with_dns(url) .await?; @@ -542,8 +558,9 @@ impl MediaLoader { if let (Some(cache), ChatCompletionRequestUserMessageContentPart::ImageUrl(image_part)) = (self.cache.as_ref(), oai_content_part) && media_io_kwargs.is_none() + && let Some(url) = image_part.image_url.as_ref().map(|media| &media.url) { - let key = Self::cache_key(image_part.image_url.url.as_str()); + let key = Self::cache_key(url.as_str()); let bytes = descriptor_bytes(&rdma_descriptor); cache.lock().put(key, rdma_descriptor.clone()); tracing::debug!(url_hash = key, bytes, "[mm-cache] insert"); @@ -600,7 +617,10 @@ mod tests { let image_url = ImageUrl::from(format!("{}/llm-optimize-deploy-graphic.png", server.url())); let content_part = ChatCompletionRequestUserMessageContentPart::ImageUrl( - ChatCompletionRequestMessageContentPartImage { image_url }, + ChatCompletionRequestMessageContentPartImage { + image_url: Some(image_url), + uuid: None, + }, ); let result = loader @@ -695,7 +715,10 @@ mod tests { let url_string = format!("{}/cache-image.png", server.url()); let image_url = ImageUrl::from(url_string); let content_part = ChatCompletionRequestUserMessageContentPart::ImageUrl( - ChatCompletionRequestMessageContentPartImage { image_url }, + ChatCompletionRequestMessageContentPartImage { + image_url: Some(image_url), + uuid: None, + }, ); // First call — populates cache. @@ -800,7 +823,10 @@ mod tests { let make_part = |path: &str| { let image_url = ImageUrl::from(format!("{}{}", server.url(), path)); ChatCompletionRequestUserMessageContentPart::ImageUrl( - ChatCompletionRequestMessageContentPartImage { image_url }, + ChatCompletionRequestMessageContentPartImage { + image_url: Some(image_url), + uuid: None, + }, ) }; diff --git a/lib/llm/src/protocols/anthropic/types.rs b/lib/llm/src/protocols/anthropic/types.rs index fb81042bcf04..7419feccbc82 100644 --- a/lib/llm/src/protocols/anthropic/types.rs +++ b/lib/llm/src/protocols/anthropic/types.rs @@ -168,6 +168,7 @@ impl TryFrom for NvCreateChatCompletionRequest { /// Convert user-role content blocks into chat completion messages. /// Tool results become separate Tool messages; text/image blocks become user messages. +#[allow(deprecated)] fn convert_user_blocks( blocks: &[AnthropicContentBlock], messages: &mut Vec, @@ -198,11 +199,12 @@ fn convert_user_blocks( .map_err(|e| anyhow::anyhow!("invalid image data URI: {e}"))?; content_parts.push(ChatCompletionRequestUserMessageContentPart::ImageUrl( ChatCompletionRequestMessageContentPartImage { - image_url: ImageUrl { + image_url: Some(ImageUrl { url, detail: None, uuid: None, - }, + }), + uuid: None, }, )); } @@ -1913,7 +1915,12 @@ mod tests { // Second part: image with data URI match &parts[1] { ChatCompletionRequestUserMessageContentPart::ImageUrl(img) => { - let url_str = img.image_url.url.to_string(); + let url_str = img + .image_url + .as_ref() + .expect("converted image must contain a URL") + .url + .to_string(); assert!( url_str.starts_with("data:image/png;base64,"), "expected data URI, got: {url_str}" diff --git a/lib/llm/src/protocols/common/preprocessor.rs b/lib/llm/src/protocols/common/preprocessor.rs index 86baf1d2a73c..ab05d6c56a84 100644 --- a/lib/llm/src/protocols/common/preprocessor.rs +++ b/lib/llm/src/protocols/common/preprocessor.rs @@ -152,11 +152,16 @@ pub enum MultimodalData { #[serde(rename(serialize = "Url"))] RawUrl(String), Decoded(RdmaMediaDataDescriptor), + /// Payload-free image slot resolved by vLLM's multimodal processor cache. + UuidOnly(String), } // multimodal map containing {mm_part_type: [data...]} pub type MultimodalDataMap = std::collections::HashMap>; +/// Image cache UUIDs aligned positionally with `image_url` data slots. +pub type MultimodalUuidMap = std::collections::HashMap>>; + /// [`PreprocessedRequest`] is the internal representation of an LLM request. The [`dynamo.llm-preprocessor`] /// crate is responsible for converting request from the public APIs to this internal representation. #[derive(Serialize, Deserialize, Debug, Clone, Builder)] @@ -184,6 +189,11 @@ pub struct PreprocessedRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_modal_data: Option, + /// User-provided vLLM image cache identities aligned with `multi_modal_data`. + #[builder(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub multi_modal_uuids: Option, + /// Optional multimodal routing-only fields (separate from execution payload). #[builder(default)] #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/lib/llm/src/protocols/openai/responses/mod.rs b/lib/llm/src/protocols/openai/responses/mod.rs index f9c0d8bbce17..13ee3caa445d 100644 --- a/lib/llm/src/protocols/openai/responses/mod.rs +++ b/lib/llm/src/protocols/openai/responses/mod.rs @@ -253,6 +253,7 @@ fn convert_image_detail_str(detail: &impl serde::Serialize) -> ChatImageDetail { } /// Convert a slice of InputContent to ChatCompletionRequestUserMessageContent. +#[allow(deprecated)] fn convert_input_content_to_user_content( content: &[InputContent], ) -> Result { @@ -289,11 +290,12 @@ fn convert_input_content_to_user_content( .map_err(|e| anyhow::anyhow!("Invalid image URL '{}': {}", url_str, e))?; chat_parts.push(ChatCompletionRequestUserMessageContentPart::ImageUrl( ChatCompletionRequestMessageContentPartImage { - image_url: ImageUrl { + image_url: Some(ImageUrl { url, detail: Some(convert_image_detail_str(&img.detail)), uuid: None, - }, + }), + uuid: None, }, )); } diff --git a/lib/llm/tests/preprocessor.rs b/lib/llm/tests/preprocessor.rs index 0921a13e765d..cea9b5137aad 100644 --- a/lib/llm/tests/preprocessor.rs +++ b/lib/llm/tests/preprocessor.rs @@ -631,6 +631,179 @@ async fn test_media_url_passthrough(#[case] media_chunks: &[(&str, usize)]) { } } +mod cached_multimodal_uuid { + use std::sync::Arc; + + use super::Request; + use dynamo_llm::model_card::ModelDeploymentCard; + use dynamo_llm::preprocessor::OpenAIPreprocessor; + use dynamo_llm::protocols::common::preprocessor::MultimodalData; + + const MODEL_PATH: &str = "tests/data/sample-models/mock-llama-3.1-8b-instruct"; + + fn make_preprocessor() -> Arc { + let mut mdc = ModelDeploymentCard::load_from_disk(MODEL_PATH, None).unwrap(); + mdc.set_name("test-model"); + OpenAIPreprocessor::new(mdc).unwrap() + } + + #[tokio::test] + async fn preserves_url_and_uuid_only_slot_alignment() { + let messages = r#"[ + { + "role": "user", + "content": [ + {"type": "text", "text": "compare"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/first.png"}, + "uuid": "image-a" + }, + { + "type": "image_url", + "image_url": null, + "uuid": "image-b" + } + ] + } + ]"#; + let request = Request::from(messages, None, None, "test-model".to_string()); + + let (preprocessed, _, _) = make_preprocessor() + .preprocess_request(&request, None) + .await + .unwrap(); + + let images = &preprocessed.multi_modal_data.as_ref().unwrap()["image_url"]; + assert_eq!(images.len(), 2); + assert!(matches!(images[0], MultimodalData::Url(_))); + assert!(matches!( + &images[1], + MultimodalData::UuidOnly(value) if value == "image-b" + )); + assert_eq!( + preprocessed.multi_modal_uuids.as_ref().unwrap()["image_url"], + vec![Some("image-a".to_string()), Some("image-b".to_string())] + ); + } + + #[tokio::test] + async fn accepts_deprecated_nested_image_uuid() { + let messages = r#"[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/first.png", + "uuid": "92b888ad-e64a-478f-b688-5091e16544e3" + } + } + ] + } + ]"#; + let request = Request::from(messages, None, None, "test-model".to_string()); + + let (preprocessed, _, _) = make_preprocessor() + .preprocess_request(&request, None) + .await + .unwrap(); + + assert_eq!( + preprocessed.multi_modal_uuids.as_ref().unwrap()["image_url"], + vec![Some("92b888ad-e64a-478f-b688-5091e16544e3".to_string())] + ); + } + + #[tokio::test] + async fn rejects_audio_and_video_cache_uuids() { + for messages in [ + r#"[{ + "role": "user", + "content": [{ + "type": "video_url", + "video_url": {"url": "https://example.com/video.mp4"}, + "uuid": "cached-video" + }] + }]"#, + r#"[{ + "role": "user", + "content": [{ + "type": "audio_url", + "audio_url": { + "url": "https://example.com/audio.wav", + "uuid": "92b888ad-e64a-478f-b688-5091e16544e3" + } + }] + }]"#, + ] { + let request = Request::from(messages, None, None, "test-model".to_string()); + let error = make_preprocessor() + .preprocess_request(&request, None) + .await + .expect_err("audio and video cache UUIDs must be rejected"); + + assert!( + format!("{error:#}").contains("supported only for image_url parts with vLLM"), + "unexpected error: {error:#}" + ); + } + } + + #[tokio::test] + async fn rejects_media_part_without_url_or_uuid() { + let messages = r#"[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": null} + ] + } + ]"#; + let request = Request::from(messages, None, None, "test-model".to_string()); + + let error = make_preprocessor() + .preprocess_request(&request, None) + .await + .expect_err("media without a URL or UUID must be rejected"); + + let error_chain = format!("{error:#}"); + assert!( + error_chain.contains("neither `url` nor `uuid`"), + "unexpected error: {error_chain}" + ); + } + + #[tokio::test] + async fn rejects_empty_media_uuid() { + let messages = r#"[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + "uuid": "" + } + ] + } + ]"#; + let request = Request::from(messages, None, None, "test-model".to_string()); + + let error = make_preprocessor() + .preprocess_request(&request, None) + .await + .expect_err("an empty media UUID must be rejected"); + + let error_chain = format!("{error:#}"); + assert!( + error_chain.contains("uuid must be a non-empty string"), + "unexpected error: {error_chain}" + ); + } +} + mod context_length_validation { use dynamo_llm::model_card::ModelDeploymentCard; use dynamo_llm::preprocessor::OpenAIPreprocessor; diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py index 8d061656c9f1..1e671c1764b3 100644 --- a/tests/serve/multimodal_profiles/vllm.py +++ b/tests/serve/multimodal_profiles/vllm.py @@ -15,6 +15,7 @@ make_image_payload, make_image_payload_b64, make_image_payload_cached_tokens, + make_image_payload_uuid_passthrough, make_video_payload, ) from tests.utils.payload_builder import chat_payload, chat_payload_default @@ -415,7 +416,31 @@ timeout_s=670, profiled_vram_gib=12.0, requested_vllm_kv_cache_bytes=922_354_000, - tests=[MmCase(payload=make_image_payload(["green"]))], + tests=[ + MmCase(payload=make_image_payload(["green"])), + MmCase( + suffix="uuid_passthrough", + payload=make_image_payload_uuid_passthrough( + ["green"], exercise_embedding_cache=True + ), + extra_script_args=[ + "--mm-processor-cache-gb", + "4", + "--multimodal-embedding-cache-capacity-gb", + "1", + # Gemma 4 budgets 280 embeddings per image; the + # 512x512 fixture emits 256. A 280-slot GPU cache can + # retain only one, so the second fill evicts the first. + "--max-num-batched-tokens", + "280", + "--limit-mm-per-prompt", + '{"image": 1, "video": 0, "audio": 0}', + ], + # The connector runs in vLLM's spawned EngineCore, so + # its debug hit diagnostic uses vLLM's logger level. + env={"VLLM_LOGGING_LEVEL": "DEBUG"}, + ), + ], ), }, extra_vllm_args=["--dtype", "bfloat16"], diff --git a/tests/utils/client.py b/tests/utils/client.py index 355c7f008de7..d4b472180202 100644 --- a/tests/utils/client.py +++ b/tests/utils/client.py @@ -39,8 +39,10 @@ def _sanitize_payload_for_logging(payload: Dict[str, Any]) -> Dict[str, Any]: if isinstance(content, list): for part in content: if isinstance(part, dict) and part.get("type") == "image_url": - image_url = part.get("image_url", {}) - if "url" in image_url: + image_url = part.get("image_url") + if isinstance(image_url, dict) and isinstance( + image_url.get("url"), str + ): image_url["url"] = _truncate_base64_url(image_url["url"]) return sanitized diff --git a/tests/utils/multimodal.py b/tests/utils/multimodal.py index 374785212eee..87cd66592aa7 100644 --- a/tests/utils/multimodal.py +++ b/tests/utils/multimodal.py @@ -3,6 +3,8 @@ import base64 import os +import re +from copy import deepcopy from dataclasses import dataclass, field from pathlib import Path from typing import Any, Optional, Type @@ -133,6 +135,139 @@ def make_image_payload_cached_tokens( ) +class UuidPassthroughChatPayload(ChatPayload): + """Send a URL+UUID cache fill followed by a UUID-only cache hit. + + When ``exercise_embedding_cache`` is true, insert a second URL+UUID fill + with a different UUID. Tests can pair this sequence with a one-image GPU + encoder cache so the final request must load the first embedding from + Dynamo's CPU embedding cache. + """ + + def __init__( + self, + *, + expected_response: list[str], + image_uuid: str = "dynamo-mm-cache-image-1", + max_tokens: int = 100, + temperature: float = 0.0, + timeout: int = 60, + exercise_embedding_cache: bool = False, + ): + object.__setattr__(self, "_uuid_initializing", True) + object.__setattr__(self, "_uuid_model", None) + object.__setattr__(self, "_uuid_image_uuid", image_uuid) + object.__setattr__( + self, + "_uuid_eviction_image_uuid", + f"{image_uuid}-eviction" if exercise_embedding_cache else None, + ) + object.__setattr__( + self, + "_uuid_final_expected_log", + [ + "Dynamo multimodal embedding cache hit: " + f"identifier={re.escape(repr(image_uuid))}" + ] + if exercise_embedding_cache + else [], + ) + object.__setattr__(self, "_uuid_max_tokens", max_tokens) + object.__setattr__(self, "_uuid_temperature", temperature) + object.__setattr__(self, "_uuid_request_index", 0) + object.__setattr__(self, "_uuid_body_storage", None) + super().__init__( + body=None, + expected_response=expected_response, + expected_log=[], + repeat_count=3 if exercise_embedding_cache else 2, + timeout=timeout, + ) + object.__setattr__(self, "_uuid_initializing", False) + + def with_model(self, model): + payload = deepcopy(self) + object.__setattr__(payload, "_uuid_model", model) + object.__setattr__(payload, "_uuid_request_index", 0) + object.__setattr__(payload, "expected_log", []) + return payload + + @property + def body(self) -> dict: # type: ignore[override] + request_index = self._uuid_request_index + object.__setattr__(self, "_uuid_request_index", request_index + 1) + + eviction_image_uuid = self._uuid_eviction_image_uuid + if request_index == 0: + image_url = {"url": MULTIMODAL_IMG_URL} + image_uuid = self._uuid_image_uuid + text = _MULTIMODAL_COLOR_PROMPT + elif request_index == 1 and eviction_image_uuid is not None: + image_url = {"url": MULTIMODAL_IMG_URL} + image_uuid = eviction_image_uuid + text = _MULTIMODAL_COLOR_PROMPT + else: + image_url = None + image_uuid = self._uuid_image_uuid + text = ( + "What colors are prominent in the same image? " + "Respond only with the colors." + ) + + if request_index == self.repeat_count - 1: + object.__setattr__( + self, + "expected_log", + list(self._uuid_final_expected_log), + ) + + body = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": text}, + { + "type": "image_url", + "image_url": image_url, + "uuid": image_uuid, + }, + ], + } + ], + "max_tokens": self._uuid_max_tokens, + "temperature": self._uuid_temperature, + "stream": False, + } + if self._uuid_model is not None: + body["model"] = self._uuid_model + return body + + @body.setter + def body(self, value) -> None: + object.__setattr__(self, "_uuid_body_storage", value) + if not getattr(self, "_uuid_initializing", True): + object.__setattr__(self, "_uuid_request_index", 0) + object.__setattr__(self, "expected_log", []) + + def final_validation(self) -> None: + assert ( + self._uuid_request_index == self.repeat_count + ), "UUID passthrough payload did not send the expected request sequence" + + +def make_image_payload_uuid_passthrough( + expected_response: list[str], + *, + exercise_embedding_cache: bool = False, +) -> ChatPayload: + """Build the maintained vLLM cached multimodal UUID smoke payload.""" + return UuidPassthroughChatPayload( + expected_response=expected_response, + exercise_embedding_cache=exercise_embedding_cache, + ) + + class Base64LazyChatPayload(ChatPayload): """ChatPayload variant that defers reading the multimodal test PNG until the first `.body` access. diff --git a/tests/utils/test_client.py b/tests/utils/test_client.py new file mode 100644 index 000000000000..ce1ee5d712bf --- /dev/null +++ b/tests/utils/test_client.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from tests.utils.client import _sanitize_payload_for_logging + +pytestmark = [ + pytest.mark.unit, + pytest.mark.pre_merge, + pytest.mark.gpu_0, +] + + +def test_sanitize_payload_preserves_uuid_only_image_part() -> None: + payload = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": None, + "uuid": "cached-image", + } + ], + } + ] + } + + assert _sanitize_payload_for_logging(payload) == payload diff --git a/tests/utils/test_multimodal.py b/tests/utils/test_multimodal.py new file mode 100644 index 000000000000..0484cef2884e --- /dev/null +++ b/tests/utils/test_multimodal.py @@ -0,0 +1,73 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import re + +import pytest + +from tests.serve.conftest import MULTIMODAL_IMG_URL +from tests.utils.multimodal import UuidPassthroughChatPayload + +pytestmark = [ + pytest.mark.unit, + pytest.mark.pre_merge, + pytest.mark.gpu_0, +] + + +def _image_part(body: dict) -> dict: + return body["messages"][0]["content"][1] + + +def test_uuid_passthrough_payload_sends_fill_then_uuid_only() -> None: + payload = UuidPassthroughChatPayload(expected_response=["green"]) + + fill = _image_part(payload.body) + reuse = _image_part(payload.body) + + assert fill == { + "type": "image_url", + "image_url": {"url": MULTIMODAL_IMG_URL}, + "uuid": "dynamo-mm-cache-image-1", + } + assert reuse == { + "type": "image_url", + "image_url": None, + "uuid": "dynamo-mm-cache-image-1", + } + assert payload.expected_log == [] + payload.final_validation() + + +def test_uuid_embedding_cache_payload_checks_hit_after_gpu_eviction() -> None: + payload = UuidPassthroughChatPayload( + expected_response=["green"], + exercise_embedding_cache=True, + ) + + first_fill = _image_part(payload.body) + assert first_fill["uuid"] == "dynamo-mm-cache-image-1" + assert first_fill["image_url"] == {"url": MULTIMODAL_IMG_URL} + assert payload.expected_log == [] + + eviction_fill = _image_part(payload.body) + assert eviction_fill["uuid"] == "dynamo-mm-cache-image-1-eviction" + assert eviction_fill["image_url"] == {"url": MULTIMODAL_IMG_URL} + assert payload.expected_log == [] + + reuse = _image_part(payload.body) + assert reuse["uuid"] == "dynamo-mm-cache-image-1" + assert reuse["image_url"] is None + assert payload.expected_log == [ + "Dynamo multimodal embedding cache hit: " + r"identifier='dynamo\-mm\-cache\-image\-1'" + ] + hit_pattern = re.compile(payload.expected_log[0]) + assert hit_pattern.search( + "Dynamo multimodal embedding cache hit: " "identifier='dynamo-mm-cache-image-1'" + ) + assert not hit_pattern.search( + "Dynamo multimodal embedding cache hit: " + "identifier='dynamo-mm-cache-image-1-eviction'" + ) + payload.final_validation()