From 8ca4758f2c625d6ce20f55cae5506e161822a672 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Wed, 15 Jul 2026 20:49:36 -0700 Subject: [PATCH 1/4] feat(frontend): pass through cached multimodal UUIDs Preserve user-provided cached multimodal UUIDs across Rust and Python preprocessing, backend loading, routing metadata, and vLLM cache reuse. Co-authored-by: Indrajit Bhosale Signed-off-by: Qi Wang --- Cargo.lock | 4 +- Cargo.toml | 6 +- .../dynamo/common/multimodal/audio_loader.py | 20 +- .../dynamo/common/multimodal/image_loader.py | 21 +- .../dynamo/common/multimodal/video_loader.py | 25 ++- .../tests/multimodal/test_audio_loader.py | 24 ++ .../tests/multimodal/test_image_loader.py | 20 ++ .../tests/multimodal/test_video_loader.py | 30 +++ .../src/dynamo/frontend/sglang_processor.py | 7 +- .../frontend/tests/test_multimodal_utils.py | 102 +++++++-- .../tests/test_vllm_processor_unit.py | 90 ++++++++ components/src/dynamo/frontend/utils.py | 43 ++-- .../src/dynamo/frontend/vllm_processor.py | 66 ++++-- .../multimodal_utils/request_processor.py | 134 +++++++++-- .../test_vllm_request_processor.py | 211 +++++++++++++++++- docs/features/multimodal/multimodal-vllm.md | 45 ++++ lib/bindings/kvbm/Cargo.lock | 4 +- lib/bindings/kvbm/Cargo.toml | 4 + lib/bindings/python/Cargo.lock | 4 +- lib/bindings/python/Cargo.toml | 4 + lib/llm/src/entrypoint/input/common.rs | 2 +- lib/llm/src/preprocessor.rs | 152 ++++++++----- lib/llm/src/preprocessor/media/loader.rs | 50 ++++- lib/llm/src/protocols/anthropic/types.rs | 8 +- lib/llm/src/protocols/common/preprocessor.rs | 10 + lib/llm/src/protocols/openai/responses/mod.rs | 8 +- lib/llm/tests/preprocessor.rs | 109 +++++++++ tests/serve/multimodal_profiles/vllm.py | 10 +- tests/utils/client.py | 6 +- tests/utils/multimodal.py | 90 ++++++++ tests/utils/test_client.py | 31 +++ 31 files changed, 1171 insertions(+), 169 deletions(-) create mode 100644 tests/utils/test_client.py diff --git a/Cargo.lock b/Cargo.lock index a4934a498bd2..9cbdca86d55d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2759,8 +2759,7 @@ dependencies = [ [[package]] name = "dynamo-protocols" version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1da866e573424e374c83d64dbf7807547054ebadcd85ddb2f9f67db06bf386" +source = "git+https://github.com/furionw/frontend-crates.git?rev=31cd250192dbca6b3005bec8170e6f95d2bc0abf#31cd250192dbca6b3005bec8170e6f95d2bc0abf" dependencies = [ "async-openai", "derive_builder", @@ -2770,7 +2769,6 @@ dependencies = [ "thiserror 2.0.18", "tracing", "url", - "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 66bf1cdf63ec..66aae9bae2a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ dynamo-kv-hashing = { path = "lib/kv-hashing", version = "1.3.0" } dynamo-memory = { path = "lib/memory", version = "1.3.0" } dynamo-mocker = { path = "lib/mocker", version = "1.3.0" } dynamo-kv-router = { path = "lib/kv-router", version = "1.3.0", features = ["metrics", "runtime-protocols"] } -dynamo-protocols = { version = "=2.0.2" } +dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" } dynamo-parsers = { version = "5.0.0" } # Published on crates.io. To see all available versions: # web: https://crates.io/crates/dynamo-parsers-v2/versions @@ -180,6 +180,10 @@ tokio-rustls = { version = "0.26" } rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } rustls-pemfile = { version = "2" } +[patch.crates-io] +# Temporary until ai-dynamo/frontend-crates#119 is released. +dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" } + [profile.dev.package] insta.opt-level = 3 diff --git a/components/src/dynamo/common/multimodal/audio_loader.py b/components/src/dynamo/common/multimodal/audio_loader.py index c0095f767a3f..559755360d7b 100644 --- a/components/src/dynamo/common/multimodal/audio_loader.py +++ b/components/src/dynamo/common/multimodal/audio_loader.py @@ -18,6 +18,7 @@ # Constants for multimodal data variants URL_VARIANT_KEY: Final = "Url" DECODED_VARIANT_KEY: Final = "Decoded" +UUID_ONLY_VARIANT_KEY: Final = "UuidOnly" def _create_nixl_connector() -> Any: @@ -170,7 +171,7 @@ async def _load_decoded_audio( async def load_audio_batch( self, audio_mm_items: List[Dict[str, Any]], - ) -> List[tuple[np.ndarray, float]]: + ) -> List[tuple[np.ndarray, float] | None]: """Load a batch of audio files from multimodal data items. Supports two paths: @@ -182,31 +183,42 @@ async def load_audio_batch( List of (waveform, sample_rate) tuples. """ audio_futures: List[Awaitable[tuple[np.ndarray, float]]] = [] + slot_to_future_idx: list[int | None] = [] for idx, item in enumerate(audio_mm_items): if isinstance(item, dict) and URL_VARIANT_KEY in item: url = item[URL_VARIANT_KEY] + slot_to_future_idx.append(len(audio_futures)) audio_futures.append(self.load_audio(url)) logger.debug("Preparing to load audio from URL: %s...", url[:80]) elif isinstance(item, dict) and DECODED_VARIANT_KEY in item: if self._enable_frontend_decoding: metadata = item[DECODED_VARIANT_KEY] + slot_to_future_idx.append(len(audio_futures)) audio_futures.append(self._load_decoded_audio(metadata)) else: raise ValueError( "Received decoded audio data but enable_frontend_decoding=False. " "Enable frontend decoding to transfer decoded audio via NIXL." ) + elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item: + slot_to_future_idx.append(None) else: raise ValueError( f"Invalid audio multimodal item at index {idx}. " - "Expected dict with 'Url' or 'Decoded' key." + "Expected dict with 'Url', 'Decoded', or 'UuidOnly' key." ) results = await asyncio.gather(*audio_futures, return_exceptions=True) - loaded_audio: list[tuple[np.ndarray, float]] = [] + loaded_audio: list[tuple[np.ndarray, float] | None] = [] collective_exceptions: list[str] = [] - for media_item, result in zip(audio_mm_items, results, strict=True): + for media_item, future_idx in zip( + audio_mm_items, slot_to_future_idx, strict=True + ): + if future_idx is None: + loaded_audio.append(None) + continue + result = results[future_idx] if isinstance(result, BaseException): if isinstance(result, asyncio.CancelledError): raise result diff --git a/components/src/dynamo/common/multimodal/image_loader.py b/components/src/dynamo/common/multimodal/image_loader.py index c23d5c9e25f5..ac4c4fe7de9c 100644 --- a/components/src/dynamo/common/multimodal/image_loader.py +++ b/components/src/dynamo/common/multimodal/image_loader.py @@ -28,6 +28,7 @@ # Constants for multimodal data variants URL_VARIANT_KEY: Final = "Url" DECODED_VARIANT_KEY: Final = "Decoded" +UUID_ONLY_VARIANT_KEY: Final = "UuidOnly" def _create_nixl_connector() -> Any: @@ -268,11 +269,13 @@ async def load_image_batch( ValueError: If enable_frontend_decoding=True but nixl_connector is None """ image_futures = [] + slot_to_future_idx: list[int | None] = [] - for item in image_mm_items: + for idx, item in enumerate(image_mm_items): if isinstance(item, dict) and URL_VARIANT_KEY in item: # URL path: download and decode in Python backend url = item[URL_VARIANT_KEY] + slot_to_future_idx.append(len(image_futures)) image_futures.append(self.load_image(url)) logger.debug(f"Preparing to load image from URL: {url[:80]}...") elif isinstance(item, dict) and DECODED_VARIANT_KEY in item: @@ -280,6 +283,7 @@ async def load_image_batch( metadata = item[DECODED_VARIANT_KEY] if self._nixl_connector is None: raise RuntimeError("NIXL connector is not initialized") + slot_to_future_idx.append(len(image_futures)) image_futures.append(self._read_and_convert_nixl_image(metadata)) else: logger.error( @@ -287,6 +291,13 @@ async def load_image_batch( "Set enable_frontend_decoding=True to enable NIXL RDMA image transfer." ) raise ValueError("Could not load decoded media from frontend") + elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item: + slot_to_future_idx.append(None) + else: + raise ValueError( + f"Invalid image multimodal item at index {idx}. " + "Expected dict with 'Url', 'Decoded', or 'UuidOnly' key." + ) # Process images in parallel results = await asyncio.gather(*image_futures, return_exceptions=True) @@ -294,7 +305,13 @@ async def load_image_batch( collective_exceptions = "" status_error: HttpStatusError | None = None url_error: UrlValidationError | None = None - for media_item, result in zip(image_mm_items, results): + for media_item, future_idx in zip( + image_mm_items, slot_to_future_idx, strict=True + ): + if future_idx is None: + loaded_images.append(None) + continue + result = results[future_idx] if isinstance(result, Exception): source = media_item.get(URL_VARIANT_KEY, "decoded") logger.error(f"Failed to load image from {source[:80]}...: {result}") diff --git a/components/src/dynamo/common/multimodal/video_loader.py b/components/src/dynamo/common/multimodal/video_loader.py index a5b18590a1cb..e8247b081a9f 100644 --- a/components/src/dynamo/common/multimodal/video_loader.py +++ b/components/src/dynamo/common/multimodal/video_loader.py @@ -30,6 +30,7 @@ URL_VARIANT_KEY: Final = "Url" DECODED_VARIANT_KEY: Final = "Decoded" +UUID_ONLY_VARIANT_KEY: Final = "UuidOnly" def _create_nixl_connector() -> Any: @@ -161,28 +162,44 @@ async def _load_decoded_video( async def load_video_batch( self, video_mm_items: List[Dict[str, Any]], - ) -> List[tuple[np.ndarray, Dict[str, Any]]]: + ) -> List[tuple[np.ndarray, Dict[str, Any]] | None]: video_futures: List[Awaitable[tuple[np.ndarray, Dict[str, Any]]]] = [] + slot_to_future_idx: list[int | None] = [] - for item in video_mm_items: + for idx, item in enumerate(video_mm_items): if isinstance(item, dict) and URL_VARIANT_KEY in item: url = item[URL_VARIANT_KEY] + slot_to_future_idx.append(len(video_futures)) video_futures.append(self.load_video(url)) logger.debug("Preparing to load video from URL: %s...", url[:80]) elif isinstance(item, dict) and DECODED_VARIANT_KEY in item: if self._enable_frontend_decoding: metadata = item[DECODED_VARIANT_KEY] + slot_to_future_idx.append(len(video_futures)) video_futures.append(self._load_decoded_video(metadata)) else: raise ValueError( "Received decoded video data but enable_frontend_decoding=False. " "Enable frontend decoding to transfer decoded video frames via NIXL." ) + elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item: + slot_to_future_idx.append(None) + else: + raise ValueError( + f"Invalid video multimodal item at index {idx}. " + "Expected dict with 'Url', 'Decoded', or 'UuidOnly' key." + ) results = await asyncio.gather(*video_futures, return_exceptions=True) - loaded_videos: list[tuple[np.ndarray, Dict[str, Any]]] = [] + loaded_videos: list[tuple[np.ndarray, Dict[str, Any]] | None] = [] collective_exceptions: list[str] = [] - for media_item, result in zip(video_mm_items, results): + for media_item, future_idx in zip( + video_mm_items, slot_to_future_idx, strict=True + ): + if future_idx is None: + loaded_videos.append(None) + continue + result = results[future_idx] if isinstance(result, BaseException): if isinstance(result, asyncio.CancelledError): raise result diff --git a/components/src/dynamo/common/tests/multimodal/test_audio_loader.py b/components/src/dynamo/common/tests/multimodal/test_audio_loader.py index 5a6eb4a188cc..8e6ec3efb67e 100644 --- a/components/src/dynamo/common/tests/multimodal/test_audio_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_audio_loader.py @@ -136,3 +136,27 @@ async def test_load_audio_batch_reads_decoded_variant(monkeypatch): decoded_item, return_metadata=True, ) + + +@pytest.mark.asyncio +async def test_uuid_only_audio_slots_preserve_batch_alignment(): + loader = AudioLoader() + first = (np.zeros(8, dtype=np.float32), 16000.0) + second = (np.ones(8, dtype=np.float32), 44100.0) + loader.load_audio = AsyncMock( # type: ignore[method-assign] + side_effect=[first, second] + ) + + audios = await loader.load_audio_batch( + [ + {"Url": "https://example.com/first.wav"}, + {"UuidOnly": "cached-audio"}, + {"Url": "https://example.com/second.wav"}, + ] + ) + + assert audios[0] is not None + assert audios[1] is None + assert audios[2] is not None + np.testing.assert_array_equal(audios[0][0], first[0]) + np.testing.assert_array_equal(audios[2][0], second[0]) diff --git a/components/src/dynamo/common/tests/multimodal/test_image_loader.py b/components/src/dynamo/common/tests/multimodal/test_image_loader.py index a745e3b07cb3..455f61b73595 100644 --- a/components/src/dynamo/common/tests/multimodal/test_image_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_image_loader.py @@ -260,6 +260,26 @@ async def test_unsupported_format_batch_url_raises_415(loader: ImageLoader) -> N assert exc_info.value.status == 415 +async def test_uuid_only_image_slots_preserve_batch_alignment( + loader: ImageLoader, +) -> None: + first = Image.new("RGB", (1, 1), color="red") + second = Image.new("RGB", (1, 1), color="blue") + loader.load_image = AsyncMock( # type: ignore[method-assign] + side_effect=[first, second] + ) + + images = await loader.load_image_batch( + [ + {"Url": "https://example.com/first.png"}, + {"UuidOnly": "cached-image"}, + {"Url": "https://example.com/second.png"}, + ] + ) + + assert images == [first, None, second] + + async def test_unsupported_format_batch_data_url_raises_415( loader: ImageLoader, ) -> None: diff --git a/components/src/dynamo/common/tests/multimodal/test_video_loader.py b/components/src/dynamo/common/tests/multimodal/test_video_loader.py index 7158cb80a5bc..fe9a353273ff 100644 --- a/components/src/dynamo/common/tests/multimodal/test_video_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_video_loader.py @@ -109,3 +109,33 @@ async def test_load_video_batch_reads_decoded_variant_with_metadata(monkeypatch) decoded_item, return_metadata=True, ) + + +@pytest.mark.asyncio +async def test_uuid_only_video_slots_preserve_batch_alignment(): + loader = VideoLoader() + first = ( + np.zeros((1, 2, 2, 3), dtype=np.uint8), + {"fps": 2.0, "frames_indices": [0], "total_num_frames": 1}, + ) + second = ( + np.ones((1, 2, 2, 3), dtype=np.uint8), + {"fps": 2.0, "frames_indices": [0], "total_num_frames": 1}, + ) + loader.load_video = AsyncMock( # type: ignore[method-assign] + side_effect=[first, second] + ) + + videos = await loader.load_video_batch( + [ + {"Url": "https://example.com/first.mp4"}, + {"UuidOnly": "cached-video"}, + {"Url": "https://example.com/second.mp4"}, + ] + ) + + assert videos[0] is not None + assert videos[1] is None + assert videos[2] is not None + np.testing.assert_array_equal(videos[0][0], first[0]) + np.testing.assert_array_equal(videos[2][0], second[0]) diff --git a/components/src/dynamo/frontend/sglang_processor.py b/components/src/dynamo/frontend/sglang_processor.py index b72096b9dbfd..91cc9901ae03 100644 --- a/components/src/dynamo/frontend/sglang_processor.py +++ b/components/src/dynamo/frontend/sglang_processor.py @@ -324,9 +324,11 @@ def _build_dynamo_preproc( } # Forward multimodal URLs so the backend handler can load the media. - mm_data = extract_mm_urls(request.get("messages", [])) + mm_data, mm_uuids = extract_mm_urls(request.get("messages", [])) if mm_data: preproc["multi_modal_data"] = mm_data + if mm_uuids: + preproc["multi_modal_uuids"] = mm_uuids nvext = request.get("nvext") or {} nvext_passthrough = { @@ -575,7 +577,8 @@ async def _generate_and_stream( cumulative_output_tokens = 0 # Rust postprocessor is bypassed on this path, so emit the multimodal # content-part counts here too (else frontend metrics report zero media). - _mm_counts = extract_mm_urls(request.get("messages", [])) or {} + _mm_counts, _ = extract_mm_urls(request.get("messages", [])) + _mm_counts = _mm_counts or {} image_count = len(_mm_counts.get("image_url", [])) video_count = len(_mm_counts.get("video_url", [])) audio_count = len(_mm_counts.get("audio_url", [])) diff --git a/components/src/dynamo/frontend/tests/test_multimodal_utils.py b/components/src/dynamo/frontend/tests/test_multimodal_utils.py index 9946275a3a23..9dda4f0011a0 100644 --- a/components/src/dynamo/frontend/tests/test_multimodal_utils.py +++ b/components/src/dynamo/frontend/tests/test_multimodal_utils.py @@ -16,11 +16,11 @@ def test_returns_none_for_text_only(): messages = [ {"role": "user", "content": "What is the capital of France?"}, ] - assert extract_mm_urls(messages) is None + assert extract_mm_urls(messages) == (None, None) def test_returns_none_for_empty_messages(): - assert extract_mm_urls([]) is None + assert extract_mm_urls([]) == (None, None) def test_extracts_image_urls(): @@ -37,7 +37,7 @@ def test_extracts_image_urls(): } ] result = extract_mm_urls(messages) - assert result == {"image_url": [{"Url": "https://example.com/cat.png"}]} + assert result == ({"image_url": [{"Url": "https://example.com/cat.png"}]}, None) def test_extracts_audio_urls(): @@ -54,7 +54,10 @@ def test_extracts_audio_urls(): } ] result = extract_mm_urls(messages) - assert result == {"audio_url": [{"Url": "data:audio/wav;base64,UklGRg=="}]} + assert result == ( + {"audio_url": [{"Url": "data:audio/wav;base64,UklGRg=="}]}, + None, + ) def test_extracts_video_urls(): @@ -70,7 +73,10 @@ def test_extracts_video_urls(): } ] result = extract_mm_urls(messages) - assert result == {"video_url": [{"Url": "https://example.com/clip.mp4"}]} + assert result == ( + {"video_url": [{"Url": "https://example.com/clip.mp4"}]}, + None, + ) def test_extracts_mixed_modalities(): @@ -95,11 +101,14 @@ def test_extracts_mixed_modalities(): } ] result = extract_mm_urls(messages) - assert result == { - "image_url": [{"Url": "https://example.com/img.jpg"}], - "audio_url": [{"Url": "https://example.com/audio.wav"}], - "video_url": [{"Url": "https://example.com/video.mp4"}], - } + assert result == ( + { + "image_url": [{"Url": "https://example.com/img.jpg"}], + "audio_url": [{"Url": "https://example.com/audio.wav"}], + "video_url": [{"Url": "https://example.com/video.mp4"}], + }, + None, + ) def test_extracts_multiple_items_per_modality(): @@ -120,12 +129,15 @@ def test_extracts_multiple_items_per_modality(): } ] result = extract_mm_urls(messages) - assert result == { - "image_url": [ - {"Url": "https://example.com/a.png"}, - {"Url": "https://example.com/b.png"}, - ] - } + assert result == ( + { + "image_url": [ + {"Url": "https://example.com/a.png"}, + {"Url": "https://example.com/b.png"}, + ] + }, + None, + ) def test_ignores_non_user_messages(): @@ -142,7 +154,7 @@ def test_ignores_non_user_messages(): }, {"role": "user", "content": "Hello"}, ] - assert extract_mm_urls(messages) is None + assert extract_mm_urls(messages) == (None, None) def test_handles_malformed_content_non_dict(): @@ -162,4 +174,58 @@ def test_handles_malformed_content_non_dict(): } ] result = extract_mm_urls(messages) - assert result == {"image_url": [{"Url": "https://example.com/ok.png"}]} + assert result == ({"image_url": [{"Url": "https://example.com/ok.png"}]}, None) + + +def test_extracts_url_and_uuid_only_slots_with_alignment(): + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + "uuid": "image-a", + }, + { + "type": "image_url", + "image_url": None, + "uuid": "image-b", + }, + { + "type": "image_url", + "image_url": {"url": "https://example.com/c.png"}, + }, + ], + } + ] + + assert extract_mm_urls(messages) == ( + { + "image_url": [ + {"Url": "https://example.com/a.png"}, + {"UuidOnly": "image-b"}, + {"Url": "https://example.com/c.png"}, + ] + }, + {"image_url": ["image-a", "image-b", None]}, + ) + + +@pytest.mark.parametrize( + "part", + [ + {"type": "image_url", "image_url": None}, + {"type": "image_url", "image_url": {}}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/a.png"}, + "uuid": "", + }, + ], +) +def test_rejects_media_parts_without_valid_url_or_uuid(part): + messages = [{"role": "user", "content": [part]}] + + with pytest.raises(ValueError, match="URL or uuid|non-empty string"): + extract_mm_urls(messages) diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index 5ebdc355edb7..60eee67bdfac 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -402,6 +402,60 @@ def feature(modality, mm_hash, offset, length): } +@pytest.mark.asyncio +async def test_prepare_mm_routing_opaque_uuid_skips_routing_but_keeps_transfer( + vllm_processor_module, + monkeypatch, +): + def fail_routing(*args, **kwargs): + raise AssertionError("opaque user UUIDs must not be parsed as routing hashes") + + monkeypatch.setattr( + vllm_processor_module, + "build_mm_routing_info_from_features", + fail_routing, + ) + + class Sender: + async def prepare(self, features, modality): + assert len(features) == 1 + assert modality == "image" + return {"mm_kwargs_shm": {"modality": modality}}, ["cleanup"] + + processor = vllm_processor_module.VllmProcessor.__new__( + vllm_processor_module.VllmProcessor + ) + processor.block_size = 16 + processor.nixl_mm_enabled = True + processor.use_shm_transfer = True + processor._sender = Sender() + + feature_hash = "derived-from-uuid-and-processor-kwargs" + vllm_preproc = SimpleNamespace( + prompt_token_ids=list(range(16)), + mm_features=[ + SimpleNamespace( + modality="image", + mm_hash=feature_hash, + data=object(), + mm_position=SimpleNamespace(offset=0, length=16), + ) + ], + ) + dynamo_preproc = {"multi_modal_uuids": {"image_url": ["opaque-user-key"]}} + + mm_routing_info, cleanup_items, transferred = await processor._prepare_mm_routing( + vllm_preproc, + dynamo_preproc, + ) + + assert mm_routing_info is None + assert cleanup_items == ["cleanup"] + assert transferred is True + assert dynamo_preproc["extra_args"]["mm_hashes"] == [feature_hash] + assert dynamo_preproc["extra_args"]["mm_kwargs_shm"] == {"modality": "image"} + + class TestReasoningParserMetadata: def test_no_reasoning_parser_returns_none(self): from dynamo.frontend.vllm_processor import _build_reasoning_parser_metadata @@ -478,6 +532,42 @@ def test_kv_router_copies_reasoning_metadata_to_extra_args(self): } +@pytest.mark.asyncio +async def test_build_engine_inputs_preserves_multimodal_uuids( + vllm_processor_module, +): + class Renderer: + async def process_for_engine_async(self, prompt, arrival_time): + assert prompt == { + "prompt": "rendered prompt", + "prompt_token_ids": [1, 2, 3], + "multi_modal_data": {"image": [image]}, + "multi_modal_uuids": {"image": ["opaque-user-key"]}, + "cache_salt": "salt", + "mm_processor_kwargs": {"do_sample_frames": False}, + } + assert isinstance(arrival_time, float) + return {"type": "multimodal", "mm_hashes": ["opaque-user-key"]} + + image = object() + engine_inputs = await vllm_processor_module._build_engine_inputs( + Renderer(), + { + "prompt": "rendered prompt", + "multi_modal_data": {"image": [image]}, + "multi_modal_uuids": {"image": ["opaque-user-key"]}, + }, + [1, 2, 3], + cache_salt="salt", + mm_processor_kwargs={"do_sample_frames": False}, + ) + + assert engine_inputs == { + "type": "multimodal", + "mm_hashes": ["opaque-user-key"], + } + + class _FakeOutputProcessor: def __init__(self): self.request_states = {} diff --git a/components/src/dynamo/frontend/utils.py b/components/src/dynamo/frontend/utils.py index f248f8dffada..607d4df3213b 100644 --- a/components/src/dynamo/frontend/utils.py +++ b/components/src/dynamo/frontend/utils.py @@ -99,21 +99,22 @@ def __init__(self, message: str): def extract_mm_urls( messages: list[dict[str, Any]], -) -> dict[str, list[dict[str, str]]] | None: - """Extract multimodal URLs from OpenAI chat completion messages. +) -> tuple[dict[str, list[dict[str, str]]] | None, dict[str, list[str | None]] | None,]: + """Extract aligned multimodal data and cache UUIDs from chat messages. - Walks user message content arrays and collects ``image_url``, ``audio_url``, - and ``video_url`` entries. Returns them in the format expected by the - backend handler's ``_extract_multimodal_data()``:: + URL-backed parts become ``Url`` variants. Parts with no URL and an opaque + top-level ``uuid`` become ``UuidOnly`` variants for vLLM's multimodal + processor cache. UUID lists preserve modality-local slot order:: - { - "image_url": [{"Url": "https://..."}, ...], - "audio_url": [{"Url": "data:audio/wav;base64,..."}], - } + ({"image_url": [{"Url": "https://..."}, {"UuidOnly": "image-1"}]}, + {"image_url": ["image-1", "image-1"]}) - Returns ``None`` if no multimodal content is found. + The UUID map is ``None`` when no user UUID is present. A media content part + with neither a URL nor UUID is rejected instead of being silently dropped. """ mm_data: dict[str, list[dict[str, str]]] = {} + mm_uuids: dict[str, list[str | None]] = {} + any_uuid = False for msg in messages: if not isinstance(msg, dict) or msg.get("role") != "user": @@ -127,14 +128,28 @@ def extract_mm_urls( part_type = part.get("type") if part_type not in _MEDIA_CONTENT_TYPES: continue + + uuid_value = part.get("uuid") + if uuid_value is not None and ( + not isinstance(uuid_value, str) or not uuid_value + ): + raise ValueError(f"{part_type} uuid must be a non-empty string") + media_value = part.get(part_type) - if not isinstance(media_value, dict): - continue - url = media_value.get("url") + url = media_value.get("url") if isinstance(media_value, dict) else None if isinstance(url, str) and url: mm_data.setdefault(part_type, []).append({"Url": url}) + elif isinstance(uuid_value, str): + mm_data.setdefault(part_type, []).append({"UuidOnly": uuid_value}) + else: + raise ValueError( + f"{part_type} part must contain a non-empty URL or uuid" + ) + + mm_uuids.setdefault(part_type, []).append(uuid_value) + any_uuid |= uuid_value is not None - return mm_data or None + return mm_data or None, mm_uuids if any_uuid else None def make_backend_error(engine_response: dict[str, Any]) -> dict[str, Any]: diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index 8b56a9c0829c..0c52e27d1a4b 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -190,6 +190,24 @@ def _inject_routing_metadata( target["mm_routing_info"] = mm_routing_info +async def _build_engine_inputs( + renderer: Any, + engine_prompt: dict[str, Any], + prompt_token_ids: list[int], + *, + cache_salt: str | None, + mm_processor_kwargs: dict[str, Any] | None, +) -> dict[str, Any]: + """Convert a rendered chat prompt into the EngineInput vLLM expects.""" + prompt_inputs = {**engine_prompt, "prompt_token_ids": prompt_token_ids} + if cache_salt is not None: + prompt_inputs["cache_salt"] = cache_salt + if mm_processor_kwargs is not None: + prompt_inputs["mm_processor_kwargs"] = mm_processor_kwargs + + return await renderer.process_for_engine_async(prompt_inputs, time.time()) + + class VllmProcessor: def __init__( self, @@ -259,10 +277,15 @@ async def _prepare_mm_routing( rng_routing = _nvtx.start_range("mm_frontend:build_routing_info", color="cyan") if vllm_preproc.mm_features: - mm_routing_info = build_mm_routing_info_from_features( - vllm_preproc.mm_features, - prompt_token_ids=list(vllm_preproc.prompt_token_ids), - ) + # A user-provided UUID is an opaque cache identity, not necessarily a + # hexadecimal content hash. Keep these requests on text-prefix routing; + # the feature metadata is still forwarded for backend cache reuse and + # optional tensor transfer. + if not dynamo_preproc.get("multi_modal_uuids"): + mm_routing_info = build_mm_routing_info_from_features( + vllm_preproc.mm_features, + prompt_token_ids=list(vllm_preproc.prompt_token_ids), + ) ( mm_hashes_list, mm_placeholders_list, @@ -473,22 +496,20 @@ async def _generator_inner( "Logprobs requested but not supported in distributed inference mode" ) - # The renderer's process_for_engine() always returns a fully processed - # EngineInput (TokenInputs or MultiModalInputs) with a "type" key. - # Pass it directly to process_inputs() — no need to rebuild a - # TokensPrompt, and this avoids the deprecation warning. - prompt_inputs = engine_prompt - if request_for_sampling.cache_salt is not None: - prompt_inputs["cache_salt"] = request_for_sampling.cache_salt - if request_for_sampling.mm_processor_kwargs is not None: - prompt_inputs[ - "mm_processor_kwargs" - ] = request_for_sampling.mm_processor_kwargs - with _nvtx.annotate("mm_frontend:process_inputs", color="orange"): + # render_messages_async returns a raw prompt. Convert it to a typed + # EngineInput before process_inputs so vLLM preserves MM UUIDs and + # does not run its multimodal processor a second time. + engine_inputs = await _build_engine_inputs( + self.input_processor.renderer, + engine_prompt, + tokens, + cache_salt=request_for_sampling.cache_salt, + mm_processor_kwargs=request_for_sampling.mm_processor_kwargs, + ) vllm_preproc: EngineCoreRequest = self.input_processor.process_inputs( request_id, - prompt_inputs, + engine_inputs, sampling_params, GENERATION_TASKS, # vLLM 0.17.0: required supported_tasks arg ) @@ -544,6 +565,13 @@ async def _generator_inner( if reasoning_parser_kwargs is not None: dynamo_preproc["reasoning_parser_kwargs"] = reasoning_parser_kwargs + # Parse user cache identities before building routing metadata. Opaque + # UUIDs deliberately suppress multimodal exact routing while retaining + # vLLM's feature metadata for backend cache reuse and tensor transfer. + mm_data, mm_uuids = extract_mm_urls(request.get("messages") or []) + if mm_uuids: + dynamo_preproc["multi_modal_uuids"] = mm_uuids + # Extract MM routing metadata and prepare transfer. cleanup_items: list = [] try: @@ -565,7 +593,6 @@ async def _generator_inner( ) all_transferred = nixl_transferred and n_with_data == n_features if not all_transferred: - mm_data = extract_mm_urls(request.get("messages") or []) if mm_data: dynamo_preproc["multi_modal_data"] = mm_data @@ -671,7 +698,8 @@ async def _generate_and_stream( # content-part counts here too (else frontend metrics report zero media). input_tokens = len(tokens) cumulative_output_tokens = 0 - _mm_counts = extract_mm_urls(request.get("messages") or []) or {} + _mm_counts, _ = extract_mm_urls(request.get("messages") or []) + _mm_counts = _mm_counts or {} image_count = len(_mm_counts.get("image_url", [])) video_count = len(_mm_counts.get("video_url", [])) audio_count = len(_mm_counts.get("audio_url", [])) diff --git a/components/src/dynamo/vllm/multimodal_utils/request_processor.py b/components/src/dynamo/vllm/multimodal_utils/request_processor.py index 8bf8bacd7420..b66f1f865542 100644 --- a/components/src/dynamo/vllm/multimodal_utils/request_processor.py +++ b/components/src/dynamo/vllm/multimodal_utils/request_processor.py @@ -47,6 +47,7 @@ VIDEO_URL_KEY = "video_url" AUDIO_URL_KEY = "audio_url" URL_VARIANT_KEY = "Url" +UUID_ONLY_VARIANT_KEY = "UuidOnly" def pad_mm_hashes_to_64(mm_hashes: list[str]) -> list[str]: @@ -99,6 +100,56 @@ def _build_forwarded_mm_uuids( return None +def _build_user_mm_uuids( + raw_uuids: Any, + use_unified_vision_chunk: bool, +) -> Optional[dict[str, list[str | None]]]: + """Normalize user cache identities without changing opaque values.""" + if not isinstance(raw_uuids, dict): + return None + + modality_names = { + IMAGE_URL_KEY: "image", + VIDEO_URL_KEY: "video", + AUDIO_URL_KEY: "audio", + } + normalized: dict[str, list[str | None]] = {} + for modality, values in raw_uuids.items(): + if not isinstance(values, list): + continue + backend_modality = modality_names.get(str(modality), str(modality)) + backend_modality = _normalize_forwarded_mm_modality( + backend_modality, + use_unified_vision_chunk, + ) + normalized[backend_modality] = list(values) + return normalized or None + + +def _append_video_audio_uuid_slots( + mm_uuids: dict[str, list[str | None]], + request: dict[str, Any], + mm_processor_kwargs: Optional[dict[str, Any]], +) -> None: + """Align UUIDs for audio tracks that vLLM extracts from video inputs.""" + if not mm_processor_kwargs or not mm_processor_kwargs.get( + "use_audio_in_video", False + ): + return + + mm_data = request.get("multi_modal_data") + if not isinstance(mm_data, dict): + return + video_items = mm_data.get(VIDEO_URL_KEY) + if not isinstance(video_items, list) or not video_items: + return + + audio_items = mm_data.get(AUDIO_URL_KEY) + standalone_audio_count = len(audio_items) if isinstance(audio_items, list) else 0 + audio_uuids = mm_uuids.setdefault("audio", [None] * standalone_audio_count) + audio_uuids.extend([None] * len(video_items)) + + def _get_modality_extra_values( extra_args: dict[str, Any], grouped_key: str, @@ -140,6 +191,22 @@ def _placeholder_range_from_extra_arg(value: Any) -> PlaceholderRange: return PlaceholderRange(offset=offset, length=length) +def _normalize_optional_media_batch( + values: list[Any | None], + modality: str, +) -> Any: + """Shape UUID-only video/audio batches for vLLM's modality parsers.""" + if all(value is None for value in values): + return None + if any(value is None for value in values): + raise ValueError( + f"Mixed URL/decoded and UUID-only {modality} inputs are not " + "supported by vLLM; provide media for every item or use only " + "cached UUIDs for this modality." + ) + return values[0] if len(values) == 1 else values + + def compute_mm_uuids( multi_modal_data: Optional[dict[str, Any]], ) -> Optional[dict[str, list[str]]]: @@ -270,7 +337,9 @@ def validate_multimodal_request(self, request: dict[str, Any]) -> None: for key in ("mm_kwargs_shm", "mm_kwargs_nixl") ) if ( - request.get("multi_modal_data") is not None or has_transfer + request.get("multi_modal_data") is not None + or request.get("multi_modal_uuids") is not None + or has_transfer ) and not self.enable_multimodal: raise self._multimodal_disabled_error() @@ -331,7 +400,7 @@ async def extract_multimodal_data( for item in mm_map.get(IMAGE_URL_KEY, []): if isinstance(item, dict) and URL_VARIANT_KEY in item: image_urls.append(item[URL_VARIANT_KEY]) - elif isinstance(item, dict) and "Decoded" in item: + else: supported = False if supported: vllm_mm_data = ( @@ -359,35 +428,51 @@ async def extract_multimodal_data( ) else: vllm_mm_data[image_key] = ( - images[0] if len(images) == 1 else images + images[0] + if len(images) == 1 and images[0] is not None + else images ) video_items = mm_map.get(VIDEO_URL_KEY, []) if video_items: videos = await self.video_loader.load_video_batch(video_items) if videos: - vllm_mm_data["video"] = videos[0] if len(videos) == 1 else videos + vllm_mm_data["video"] = _normalize_optional_media_batch( + videos, + "video", + ) audio_items = mm_map.get(AUDIO_URL_KEY, []) if audio_items: audios = await self.audio_loader.load_audio_batch(audio_items) if audios: - vllm_mm_data["audio"] = audios[0] if len(audios) == 1 else audios + vllm_mm_data["audio"] = _normalize_optional_media_batch( + audios, + "audio", + ) if ( video_items and mm_processor_kwargs and mm_processor_kwargs.get("use_audio_in_video", False) ): + if any( + isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item + for item in audio_items + ): + raise ValueError( + "use_audio_in_video cannot be combined with UUID-only " + "standalone audio because vLLM cannot represent cached " + "and media-backed audio slots in one batch." + ) video_audios = [] for item in video_items: url = item.get(URL_VARIANT_KEY) if isinstance(item, dict) else None if not url: raise ValueError( "use_audio_in_video requires all video items to be " - "URL-based. Got a non-URL video item (e.g. frontend-" - "decoded). Audio extraction from decoded video data " - "is not yet supported." + "URL-based. UUID-only and frontend-decoded video " + "items cannot provide an audio stream." ) try: video_audios.append(await self.audio_loader.load_audio(url)) @@ -426,7 +511,10 @@ async def try_receive_mm_kwargs( if shm_meta_raw: shm_metadata = MmKwargsShmTransferMetadata.model_validate(shm_meta_raw) return await self._receive_mm_kwargs( - extra_args, "shm", MmKwargsShmReceiver(), shm_metadata + extra_args, + "shm", + MmKwargsShmReceiver(), + shm_metadata, ) if nixl_meta_raw: @@ -434,7 +522,10 @@ async def try_receive_mm_kwargs( if self._mm_kwargs_receiver is None: self._mm_kwargs_receiver = MmKwargsNixlReceiver() return await self._receive_mm_kwargs( - extra_args, "nixl", self._mm_kwargs_receiver, nixl_metadata + extra_args, + "nixl", + self._mm_kwargs_receiver, + nixl_metadata, ) except Exception: logger.exception( @@ -509,8 +600,12 @@ async def _receive_mm_kwargs( ) return None - padded_hashes = pad_mm_hashes_to_64(list(mm_hashes)) - mm_hashes_dict = {backend_modality: padded_hashes} + # These are vLLM's final feature hashes. When the request supplies + # an opaque UUID, vLLM derives this identity from the UUID together + # with mm_processor_kwargs. Any rewriting (including zero-padding) + # would create a different worker-cache key. + feature_hashes = list(mm_hashes) + mm_hashes_dict = {backend_modality: feature_hashes} mm_kwargs_dict = {backend_modality: kwargs_items} engine_input = { "type": "multimodal", @@ -549,10 +644,21 @@ def build_tokens_prompt( ) -> TokensPrompt: """Create a TokensPrompt with stable multimodal UUIDs.""" extra_args = request.get("extra_args") or {} - mm_uuids = _build_forwarded_mm_uuids( - extra_args, + mm_uuids = _build_user_mm_uuids( + request.get("multi_modal_uuids"), self.use_unified_vision_chunk, ) + if mm_uuids is not None: + _append_video_audio_uuid_slots( + mm_uuids, + request, + mm_processor_kwargs, + ) + if mm_uuids is None: + mm_uuids = _build_forwarded_mm_uuids( + extra_args, + self.use_unified_vision_chunk, + ) if mm_uuids is None and self.embedding_loader is None: mm_uuids = compute_mm_uuids(multi_modal_data) if mm_uuids is not None: diff --git a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py index 635ef83f635b..ea77d87d3e45 100644 --- a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py +++ b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py @@ -114,6 +114,26 @@ async def test_merges_encoder_images_with_local_video_and_decoded_fallback(): processor.embedding_loader.load_multimodal_embeddings.assert_awaited_once() +@pytest.mark.asyncio +async def test_extracts_uuid_only_media_as_aligned_none_slots(): + processor = _processor() + image = Image.new("RGB", (1, 1)) + image_items = [ + {"Url": "https://example.com/image.png"}, + {"UuidOnly": "cached-image"}, + ] + processor.image_loader.load_image_batch.return_value = [image, None] + + result = await processor.extract_multimodal_data( + {"multi_modal_data": {"image_url": image_items}}, + "request-cached-image", + None, + ) + + assert result == {"image": [image, None]} + processor.image_loader.load_image_batch.assert_awaited_once_with(image_items) + + @pytest.mark.asyncio async def test_rejects_media_when_multimodal_is_disabled(): processor = _processor(enabled=False) @@ -244,7 +264,7 @@ async def test_audio_in_video_preserves_order_and_merges_standalone_audio(): @pytest.mark.parametrize( ("video_item", "audio_error", "message"), [ - ({"Decoded": {"shape": [2, 4, 4, 3]}}, None, "non-URL video item"), + ({"Decoded": {"shape": [2, 4, 4, 3]}}, None, "URL-based"), ( {"Url": "https://example.com/silent.mp4"}, RuntimeError("no audio stream"), @@ -267,6 +287,84 @@ async def test_audio_in_video_rejects_unusable_audio(video_item, audio_error, me ) +@pytest.mark.asyncio +async def test_audio_in_video_rejects_uuid_only_standalone_audio(): + processor = _processor() + processor.video_loader.load_video_batch.return_value = [object()] + processor.audio_loader.load_audio_batch.return_value = [None] + + with pytest.raises(ValueError, match="UUID-only standalone audio"): + await processor.extract_multimodal_data( + { + "multi_modal_data": { + "video_url": [{"Url": "https://example.com/video.mp4"}], + "audio_url": [{"UuidOnly": "cached-audio"}], + } + }, + "request-video-cached-audio", + None, + {"use_audio_in_video": True}, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_key", "result_key", "loader_name"), + [ + ("video_url", "video", "video_loader"), + ("audio_url", "audio", "audio_loader"), + ], +) +async def test_uuid_only_video_audio_use_modality_level_none( + request_key, + result_key, + loader_name, +): + processor = _processor() + loader = getattr(processor, loader_name) + getattr(loader, f"load_{result_key}_batch").return_value = [None] + + result = await processor.extract_multimodal_data( + {"multi_modal_data": {request_key: [{"UuidOnly": "cached-key"}]}}, + "request-uuid-only", + None, + ) + + assert result == {result_key: None} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_key", "result_key", "loader_name"), + [ + ("video_url", "video", "video_loader"), + ("audio_url", "audio", "audio_loader"), + ], +) +async def test_mixed_media_and_uuid_only_video_audio_are_rejected( + request_key, + result_key, + loader_name, +): + processor = _processor() + loader = getattr(processor, loader_name) + getattr(loader, f"load_{result_key}_batch").return_value = [object(), None] + + with pytest.raises(ValueError, match=f"Mixed .* UUID-only {result_key}"): + await processor.extract_multimodal_data( + { + "multi_modal_data": { + request_key: [ + {"Url": "https://example.com/media"}, + {"UuidOnly": "cached-key"}, + ] + } + }, + "request-mixed-uuid", + None, + ) + + def test_build_tokens_prompt_forwards_hashes_kwargs_and_vision_chunk(): processor = _processor(unified_vision_chunk=True) mm_data = {"vision_chunk": {"type": "image", "image": object(), "uuid": None}} @@ -286,6 +384,74 @@ def test_build_tokens_prompt_forwards_hashes_kwargs_and_vision_chunk(): assert prompt["mm_processor_kwargs"] == {"num_crops": 4} +def test_build_tokens_prompt_prefers_opaque_user_uuids_without_padding(): + processor = _processor() + mm_data = {"image": [object(), None]} + + prompt = processor.build_tokens_prompt( + { + "token_ids": [1, 2, 3], + "multi_modal_uuids": {"image_url": ["sku-image-a", "sku-image-b"]}, + "extra_args": {"mm_hashes": ["routing-a", "routing-b"]}, + }, + mm_data, + None, + ) + + assert prompt["multi_modal_data"] is mm_data + assert prompt["multi_modal_uuids"] == {"image": ["sku-image-a", "sku-image-b"]} + + +def test_build_tokens_prompt_normalizes_all_user_uuid_modalities(): + processor = _processor(unified_vision_chunk=True) + + prompt = processor.build_tokens_prompt( + { + "token_ids": [1, 2, 3], + "multi_modal_uuids": { + "image_url": ["image-key"], + "video_url": ["video-key"], + "audio_url": ["audio-key"], + }, + }, + {"vision_chunk": [None], "video": [None], "audio": [None]}, + None, + ) + + assert prompt["multi_modal_uuids"] == { + "vision_chunk": ["image-key"], + "video": ["video-key"], + "audio": ["audio-key"], + } + + +def test_build_tokens_prompt_aligns_video_derived_audio_uuid_slots(): + processor = _processor() + standalone_audio = object() + video_audio = object() + + prompt = processor.build_tokens_prompt( + { + "token_ids": [1, 2, 3], + "multi_modal_data": { + "video_url": [{"Url": "https://example.com/video.mp4"}], + "audio_url": [{"Url": "https://example.com/audio.wav"}], + }, + "multi_modal_uuids": { + "video_url": ["video-key"], + "audio_url": [None], + }, + }, + {"video": object(), "audio": [standalone_audio, video_audio]}, + {"use_audio_in_video": True}, + ) + + assert prompt["multi_modal_uuids"] == { + "video": ["video-key"], + "audio": [None, None], + } + + def test_build_tokens_prompt_preserves_grouped_forwarded_hashes(): processor = _processor() @@ -606,12 +772,42 @@ async def test_receive_transferred_kwargs_injects_vllm_cache(monkeypatch): metadata, ) - padded_hash = "hash".ljust(64, "0") assert result is not None assert result["prompt_token_ids"] == [10, 11, 12] - assert result["mm_hashes"] == {"image": [padded_hash]} + assert result["mm_hashes"] == {"image": ["hash"]} + input_processor.inject_into_mm_cache.assert_called_once_with( + {"image": ["hash"]}, {"image": [item]} + ) + + +@pytest.mark.asyncio +async def test_receive_transferred_kwargs_keeps_vllm_feature_hash(monkeypatch): + input_processor = SimpleNamespace(inject_into_mm_cache=MagicMock()) + processor = _processor() + processor.engine_client = SimpleNamespace(input_processor=input_processor) + item = MagicMock(spec=mod.MultiModalKwargsItem) + monkeypatch.setattr(mod.pickle, "loads", lambda payload: item) + receiver = SimpleNamespace( + receive=AsyncMock(return_value={"__pickled_kwargs_item__": [b"payload"]}) + ) + + result = await processor._receive_mm_kwargs( + { + # vLLM derives this hash from the opaque user UUID together with + # mm_processor_kwargs; the raw UUID must not replace it. + "mm_hashes": ["derived-feature-hash"], + "mm_placeholders": [[1, 2]], + "expanded_token_ids": [10, 11, 12], + }, + "shm", + receiver, + SimpleNamespace(modality="image", mm_hashes=[]), + ) + + assert result is not None + assert result["mm_hashes"] == {"image": ["derived-feature-hash"]} input_processor.inject_into_mm_cache.assert_called_once_with( - {"image": [padded_hash]}, {"image": [item]} + {"image": ["derived-feature-hash"]}, {"image": [item]} ) @@ -648,13 +844,12 @@ async def test_receive_transferred_kwargs_uses_grouped_metadata_and_vision_chunk metadata, ) - padded_hash = "grouped_hash".ljust(64, "0") assert result is not None - assert result["mm_hashes"] == {"vision_chunk": [padded_hash]} + assert result["mm_hashes"] == {"vision_chunk": ["grouped_hash"]} placeholder = result["mm_placeholders"]["vision_chunk"][0] assert placeholder.get_num_embeds() == 1 input_processor.inject_into_mm_cache.assert_called_once_with( - {"vision_chunk": [padded_hash]}, {"vision_chunk": [item]} + {"vision_chunk": ["grouped_hash"]}, {"vision_chunk": [item]} ) @@ -678,7 +873,7 @@ async def test_receive_transferred_kwargs_falls_back_to_metadata_hashes(monkeypa ) assert result is not None - assert result["mm_hashes"] == {"video": ["metadata_hash".ljust(64, "0")]} + assert result["mm_hashes"] == {"video": ["metadata_hash"]} @pytest.mark.asyncio diff --git a/docs/features/multimodal/multimodal-vllm.md b/docs/features/multimodal/multimodal-vllm.md index ee4c158196ea..7cd758e9d713 100644 --- a/docs/features/multimodal/multimodal-vllm.md +++ b/docs/features/multimodal/multimodal-vllm.md @@ -102,6 +102,51 @@ curl http://localhost:8000/v1/chat/completions \ }' ``` +### Reuse vLLM Multimodal Processor Cache Entries + +vLLM can cache processed image, video, and audio inputs under a client-provided +opaque UUID. This extension is specific to vLLM; it is not part of the OpenAI +chat completions schema and is not supported by Dynamo's other backends. + +To enable the cache, pass a nonzero `--mm-processor-cache-gb` value to the vLLM +worker. Populate an entry by adding `uuid` beside the media field: + +```json +{ + "type": "image_url", + "image_url": { + "url": "https://example.com/image.png" + }, + "uuid": "catalog-image-42" +} +``` + +Reuse that entry in a later request by setting the media field to `null` and +sending the same top-level UUID: + +```json +{ + "type": "image_url", + "image_url": null, + "uuid": "catalog-image-42" +} +``` + +UUIDs are opaque nonempty strings. Do not nest `uuid` inside `image_url`, +`video_url`, or `audio_url`. A UUID-only request fails on a cache miss because +it contains no media payload to process. Dynamo also rejects a media content +part that has neither a URL nor a UUID. + +When KV-aware routing is enabled, requests with client UUIDs use text-prefix +routing. Dynamo cannot convert an opaque client key into its content-derived +multimodal routing hash without risking different cache identities at the +router and worker. + +With vLLM, a video or audio modality may be entirely URL/decoded media or +entirely UUID-only. Do not mix media-backed and UUID-only entries within one +video or audio modality batch; vLLM does not represent missing media slots in +those batch parsers. Image batches support aligned mixed slots. + **Video request:** ```bash diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index 55a9671fd25a..4c51558ed815 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -1750,8 +1750,7 @@ dependencies = [ [[package]] name = "dynamo-protocols" version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1da866e573424e374c83d64dbf7807547054ebadcd85ddb2f9f67db06bf386" +source = "git+https://github.com/furionw/frontend-crates.git?rev=31cd250192dbca6b3005bec8170e6f95d2bc0abf#31cd250192dbca6b3005bec8170e6f95d2bc0abf" dependencies = [ "async-openai", "derive_builder", @@ -1761,7 +1760,6 @@ dependencies = [ "thiserror 2.0.18", "tracing", "url", - "uuid", ] [[package]] diff --git a/lib/bindings/kvbm/Cargo.toml b/lib/bindings/kvbm/Cargo.toml index 83b549339223..dc05697999f5 100644 --- a/lib/bindings/kvbm/Cargo.toml +++ b/lib/bindings/kvbm/Cargo.toml @@ -60,3 +60,7 @@ dlpark = { version = "0.5", features = ["pyo3", "half"], optional = true } cudarc = { version = "=0.19.3", features = ["cuda-version-from-build-system", "fallback-latest"], optional = true } [dev-dependencies] + +[patch.crates-io] +# Temporary until ai-dynamo/frontend-crates#119 is released. +dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" } diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index 693347d3351d..d40e8d6a6c0e 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2363,8 +2363,7 @@ dependencies = [ [[package]] name = "dynamo-protocols" version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1da866e573424e374c83d64dbf7807547054ebadcd85ddb2f9f67db06bf386" +source = "git+https://github.com/furionw/frontend-crates.git?rev=31cd250192dbca6b3005bec8170e6f95d2bc0abf#31cd250192dbca6b3005bec8170e6f95d2bc0abf" dependencies = [ "async-openai", "derive_builder", @@ -2374,7 +2373,6 @@ dependencies = [ "thiserror 2.0.18", "tracing", "url", - "uuid", ] [[package]] diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index 82c173d884fc..0f7ff44585d2 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -118,3 +118,7 @@ tracing-subscriber = { version = "0.3", features = ["registry"] } inherits = "release" debug = true strip = false + +[patch.crates-io] +# Temporary until ai-dynamo/frontend-crates#119 is released. +dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" } diff --git a/lib/llm/src/entrypoint/input/common.rs b/lib/llm/src/entrypoint/input/common.rs index fa5b1a87dfa0..a95d014473f6 100644 --- a/lib/llm/src/entrypoint/input/common.rs +++ b/lib/llm/src/entrypoint/input/common.rs @@ -66,7 +66,7 @@ fn preprocessed_multimodal_cache_keys(request: &PreprocessedRequest) -> Vec keys.push(multimodal_cache_key_from_url(url.as_str())), MultimodalData::RawUrl(url) => keys.push(multimodal_cache_key_from_url(url)), - MultimodalData::Decoded(_) => {} + MultimodalData::Decoded(_) | MultimodalData::UuidOnly(_) => {} } } keys.sort(); diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 5f9e35e6e112..f233fc642329 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -47,7 +47,7 @@ use crate::model_card::ModelInfoType; use crate::model_card::{ModelDeploymentCard, ModelInfo}; use crate::preprocessor::media::MediaLoader; use crate::protocols::common::preprocessor::{ - MultimodalData, MultimodalDataMap, PreprocessedRequestBuilder, RoutingHints, + MultimodalData, MultimodalDataMap, MultimodalUuidMap, PreprocessedRequestBuilder, RoutingHints, }; use crate::protocols::common::timing::RequestTracker; use crate::tokenizers::Encoding; @@ -1261,8 +1261,16 @@ impl OpenAIPreprocessor { let _ = token_ids; let mut media_map: MultimodalDataMap = HashMap::new(); - let mut fetch_tasks: Vec<(String, &ChatCompletionRequestUserMessageContentPart)> = - Vec::new(); + let mut uuid_map: MultimodalUuidMap = HashMap::new(); + let mut any_user_uuid = false; + // Decoded results are written back into these reserved modality slots so + // URL-backed and UUID-only inputs retain request order. + let mut fetch_tasks: Vec<( + String, + usize, + String, + &ChatCompletionRequestUserMessageContentPart, + )> = Vec::new(); // Per-image (mm_hash, width, height) for the MM-routing path. // Accumulated in message order so we don't walk messages twice. // Cleared and returned to the caller; empty for non-image / text-only requests. @@ -1302,41 +1310,70 @@ impl OpenAIPreprocessor { _ => continue, }; for content_part in content_parts.iter() { - if has_media_loader { - let type_str = match content_part { - ChatCompletionRequestUserMessageContentPart::ImageUrl(_) => "image_url", - ChatCompletionRequestUserMessageContentPart::VideoUrl(_) => "video_url", - ChatCompletionRequestUserMessageContentPart::AudioUrl(_) => "audio_url", - _ => continue, - }; - #[cfg(feature = "mm-routing")] - if type_str == "image_url" { - total_image_count += 1; - } - fetch_tasks.push((type_str.to_string(), content_part)); - } else { - let (type_str, url) = match content_part { - ChatCompletionRequestUserMessageContentPart::ImageUrl(p) => { - ("image_url", p.image_url.url.clone()) - } - ChatCompletionRequestUserMessageContentPart::VideoUrl(p) => { - ("video_url", p.video_url.url.clone()) - } - ChatCompletionRequestUserMessageContentPart::AudioUrl(p) => { - ("audio_url", p.audio_url.url.clone()) + let (type_str, url, uuid) = match content_part { + ChatCompletionRequestUserMessageContentPart::ImageUrl(part) => ( + "image_url", + part.image_url.as_ref().and_then(|media| media.url.clone()), + part.uuid.clone(), + ), + ChatCompletionRequestUserMessageContentPart::VideoUrl(part) => ( + "video_url", + part.video_url.as_ref().and_then(|media| media.url.clone()), + part.uuid.clone(), + ), + ChatCompletionRequestUserMessageContentPart::AudioUrl(part) => ( + "audio_url", + part.audio_url.as_ref().and_then(|media| media.url.clone()), + part.uuid.clone(), + ), + _ => continue, + }; + + #[cfg(feature = "mm-routing")] + if type_str == "image_url" { + total_image_count += 1; + } + + if uuid.as_deref().is_some_and(str::is_empty) { + bail!("{type_str} uuid must be a non-empty string"); + } + + let modality = type_str.to_string(); + let slot_idx = media_map.entry(modality.clone()).or_default().len(); + any_user_uuid |= uuid.is_some(); + uuid_map + .entry(modality.clone()) + .or_default() + .push(uuid.clone()); + + match (url, uuid) { + (Some(url), _) => { + media_map + .entry(modality.clone()) + .or_default() + .push(MultimodalData::Url(url.clone())); + + if has_media_loader { + fetch_tasks.push((modality, slot_idx, url.to_string(), content_part)); + } else { + #[cfg(feature = "mm-routing")] + if type_str == "image_url" { + let mm_hash = Self::hash_image_url(url.as_str()); + url_passthrough_images.push((mm_hash, url.to_string())); + } } - _ => continue, - }; - #[cfg(feature = "mm-routing")] - if type_str == "image_url" { - total_image_count += 1; - let mm_hash = Self::hash_image_url(url.as_str()); - url_passthrough_images.push((mm_hash, url.to_string())); } - media_map - .entry(type_str.to_string()) - .or_default() - .push(MultimodalData::Url(url)); + (None, Some(uuid)) => { + media_map + .entry(modality) + .or_default() + .push(MultimodalData::UuidOnly(uuid)); + } + (None, None) => { + bail!( + "{type_str} part has neither `url` nor `uuid`; at least one is required" + ); + } } } } @@ -1345,12 +1382,15 @@ impl OpenAIPreprocessor { if !fetch_tasks.is_empty() { let loader = self.media_loader.as_ref().unwrap(); let media_io_kwargs = request.media_io_kwargs(); - let results = futures::future::join_all(fetch_tasks.iter().map(|(_, content_part)| { - loader.fetch_and_decode_media_part(content_part, media_io_kwargs) - })) - .await; + let results = + futures::future::join_all(fetch_tasks.iter().map(|(_, _, _, content_part)| { + loader.fetch_and_decode_media_part(content_part, media_io_kwargs) + })) + .await; - for ((type_str, _content_part), result) in fetch_tasks.into_iter().zip(results) { + for ((type_str, slot_idx, _source_url, _content_part), result) in + fetch_tasks.into_iter().zip(results) + { // if one item fails, errors the whole request, other items will be cleaned up by Drop let rdma_descriptor = result?; @@ -1362,14 +1402,6 @@ impl OpenAIPreprocessor { if shape.len() >= 2 { let h = shape[0] as u32; let w = shape[1] as u32; - let url_str = match _content_part { - ChatCompletionRequestUserMessageContentPart::ImageUrl(p) => { - p.image_url.url.as_str() - } - _ => unreachable!( - "rdma image_url descriptor only originates from ImageUrl content parts" - ), - }; // Frontend-decode path: hash the decoded RGB bytes so // the same image reached via different (signed) URLs // collides on the same `mm_hash` and routes to the @@ -1379,7 +1411,7 @@ impl OpenAIPreprocessor { // shouldn't happen on the frontend. let (mm_hash, hash_source) = match rdma_descriptor.content_hash() { Some(h) => (h, "decoded_bytes"), - None => (Self::hash_image_url(url_str), "url_fallback"), + None => (Self::hash_image_url(&_source_url), "url_fallback"), }; if let Some(counter) = self.image_token_counter.as_ref() { let n = counter.count_tokens(w, h); @@ -1402,10 +1434,10 @@ impl OpenAIPreprocessor { } } - media_map - .entry(type_str) - .or_default() - .push(MultimodalData::Decoded(rdma_descriptor)); + let slots = media_map + .get_mut(&type_str) + .expect("modality must exist after reserving a fetch slot"); + slots[slot_idx] = MultimodalData::Decoded(rdma_descriptor); } } @@ -1468,6 +1500,17 @@ impl OpenAIPreprocessor { if !media_map.is_empty() { builder.multi_modal_data(Some(media_map)); + if any_user_uuid { + builder.multi_modal_uuids(Some(uuid_map)); + } + + // User cache identities are opaque and cannot be converted into the + // router's canonical image hashes. Fall back to text-prefix routing + // instead of routing and publishing under different cache keys. + #[cfg(feature = "mm-routing")] + if any_user_uuid { + mm_image_entries.clear(); + } // Preserve original messages and formatted prompt in extra_args for multimodal // workers (e.g., TRT-LLM needs messages and the template-rendered prompt with @@ -1527,6 +1570,7 @@ impl OpenAIPreprocessor { // text-prefix routing. #[cfg(feature = "mm-routing")] if let Some(find_token_id) = self.routing_image_token_id + && !any_user_uuid && !mm_image_entries.is_empty() && mm_image_entries.len() == total_image_count && token_ids.iter().filter(|&&t| t == find_token_id).count() diff --git a/lib/llm/src/preprocessor/media/loader.rs b/lib/llm/src/preprocessor/media/loader.rs index 36613454f682..5c57a54e9b9e 100644 --- a/lib/llm/src/preprocessor/media/loader.rs +++ b/lib/llm/src/preprocessor/media/loader.rs @@ -472,8 +472,13 @@ impl MediaLoader { // output (resize, normalisation). When it's set we skip the // cache to stay correct; in practice it's None on the common // path so the hit rate is unaffected. - if media_io_kwargs.is_none() { - let key = Self::cache_key(image_part.image_url.url.as_str()); + if media_io_kwargs.is_none() + && let Some(url) = image_part + .image_url + .as_ref() + .and_then(|media| media.url.as_ref()) + { + let key = Self::cache_key(url.as_str()); if let Some(hit) = cache.lock().get(&key) { tracing::debug!(url_hash = key, "[mm-cache] hit"); return Ok(hit); @@ -490,7 +495,15 @@ impl MediaLoader { .as_ref() .ok_or_else(|| anyhow::anyhow!("Model does not support image inputs"))?; - let url = &image_part.image_url.url; + let url = image_part + .image_url + .as_ref() + .and_then(|media| media.url.as_ref()) + .ok_or_else(|| { + anyhow::anyhow!( + "Cannot decode an image content part without a URL; UUID-only parts must be resolved by the backend cache" + ) + })?; self.media_fetcher .check_if_url_allowed_with_dns(url) .await?; @@ -513,7 +526,15 @@ impl MediaLoader { anyhow::anyhow!("Model does not support video inputs") })?; - let url = &video_part.video_url.url; + let url = video_part + .video_url + .as_ref() + .and_then(|media| media.url.as_ref()) + .ok_or_else(|| { + anyhow::anyhow!( + "Cannot decode a video content part without a URL; UUID-only parts must be resolved by the backend cache" + ) + })?; self.media_fetcher .check_if_url_allowed_with_dns(url) .await?; @@ -542,8 +563,12 @@ impl MediaLoader { if let (Some(cache), ChatCompletionRequestUserMessageContentPart::ImageUrl(image_part)) = (self.cache.as_ref(), oai_content_part) && media_io_kwargs.is_none() + && let Some(url) = image_part + .image_url + .as_ref() + .and_then(|media| media.url.as_ref()) { - let key = Self::cache_key(image_part.image_url.url.as_str()); + let key = Self::cache_key(url.as_str()); let bytes = descriptor_bytes(&rdma_descriptor); cache.lock().put(key, rdma_descriptor.clone()); tracing::debug!(url_hash = key, bytes, "[mm-cache] insert"); @@ -600,7 +625,10 @@ mod tests { let image_url = ImageUrl::from(format!("{}/llm-optimize-deploy-graphic.png", server.url())); let content_part = ChatCompletionRequestUserMessageContentPart::ImageUrl( - ChatCompletionRequestMessageContentPartImage { image_url }, + ChatCompletionRequestMessageContentPartImage { + image_url: Some(image_url), + uuid: None, + }, ); let result = loader @@ -695,7 +723,10 @@ mod tests { let url_string = format!("{}/cache-image.png", server.url()); let image_url = ImageUrl::from(url_string); let content_part = ChatCompletionRequestUserMessageContentPart::ImageUrl( - ChatCompletionRequestMessageContentPartImage { image_url }, + ChatCompletionRequestMessageContentPartImage { + image_url: Some(image_url), + uuid: None, + }, ); // First call — populates cache. @@ -800,7 +831,10 @@ mod tests { let make_part = |path: &str| { let image_url = ImageUrl::from(format!("{}{}", server.url(), path)); ChatCompletionRequestUserMessageContentPart::ImageUrl( - ChatCompletionRequestMessageContentPartImage { image_url }, + ChatCompletionRequestMessageContentPartImage { + image_url: Some(image_url), + uuid: None, + }, ) }; diff --git a/lib/llm/src/protocols/anthropic/types.rs b/lib/llm/src/protocols/anthropic/types.rs index fb81042bcf04..e20a50871e3f 100644 --- a/lib/llm/src/protocols/anthropic/types.rs +++ b/lib/llm/src/protocols/anthropic/types.rs @@ -198,11 +198,11 @@ fn convert_user_blocks( .map_err(|e| anyhow::anyhow!("invalid image data URI: {e}"))?; content_parts.push(ChatCompletionRequestUserMessageContentPart::ImageUrl( ChatCompletionRequestMessageContentPartImage { - image_url: ImageUrl { - url, + image_url: Some(ImageUrl { + url: Some(url), detail: None, - uuid: None, - }, + }), + uuid: None, }, )); } diff --git a/lib/llm/src/protocols/common/preprocessor.rs b/lib/llm/src/protocols/common/preprocessor.rs index 86baf1d2a73c..df5fee1a4d3e 100644 --- a/lib/llm/src/protocols/common/preprocessor.rs +++ b/lib/llm/src/protocols/common/preprocessor.rs @@ -152,11 +152,16 @@ pub enum MultimodalData { #[serde(rename(serialize = "Url"))] RawUrl(String), Decoded(RdmaMediaDataDescriptor), + /// Payload-free slot resolved by the backend's multimodal processor cache. + UuidOnly(String), } // multimodal map containing {mm_part_type: [data...]} pub type MultimodalDataMap = std::collections::HashMap>; +/// Cache UUIDs aligned positionally with each modality's data slots. +pub type MultimodalUuidMap = std::collections::HashMap>>; + /// [`PreprocessedRequest`] is the internal representation of an LLM request. The [`dynamo.llm-preprocessor`] /// crate is responsible for converting request from the public APIs to this internal representation. #[derive(Serialize, Deserialize, Debug, Clone, Builder)] @@ -184,6 +189,11 @@ pub struct PreprocessedRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_modal_data: Option, + /// User-provided multimodal cache identities aligned with `multi_modal_data`. + #[builder(default)] + #[serde(default, skip_serializing_if = "Option::is_none")] + pub multi_modal_uuids: Option, + /// Optional multimodal routing-only fields (separate from execution payload). #[builder(default)] #[serde(default, skip_serializing_if = "Option::is_none")] diff --git a/lib/llm/src/protocols/openai/responses/mod.rs b/lib/llm/src/protocols/openai/responses/mod.rs index f9c0d8bbce17..f4f8a5c4652d 100644 --- a/lib/llm/src/protocols/openai/responses/mod.rs +++ b/lib/llm/src/protocols/openai/responses/mod.rs @@ -289,11 +289,11 @@ fn convert_input_content_to_user_content( .map_err(|e| anyhow::anyhow!("Invalid image URL '{}': {}", url_str, e))?; chat_parts.push(ChatCompletionRequestUserMessageContentPart::ImageUrl( ChatCompletionRequestMessageContentPartImage { - image_url: ImageUrl { - url, + image_url: Some(ImageUrl { + url: Some(url), detail: Some(convert_image_detail_str(&img.detail)), - uuid: None, - }, + }), + uuid: None, }, )); } diff --git a/lib/llm/tests/preprocessor.rs b/lib/llm/tests/preprocessor.rs index 0921a13e765d..ef648f55044b 100644 --- a/lib/llm/tests/preprocessor.rs +++ b/lib/llm/tests/preprocessor.rs @@ -631,6 +631,115 @@ async fn test_media_url_passthrough(#[case] media_chunks: &[(&str, usize)]) { } } +mod cached_multimodal_uuid { + use std::sync::Arc; + + use super::Request; + use dynamo_llm::model_card::ModelDeploymentCard; + use dynamo_llm::preprocessor::OpenAIPreprocessor; + use dynamo_llm::protocols::common::preprocessor::MultimodalData; + + const MODEL_PATH: &str = "tests/data/sample-models/mock-llama-3.1-8b-instruct"; + + fn make_preprocessor() -> Arc { + let mut mdc = ModelDeploymentCard::load_from_disk(MODEL_PATH, None).unwrap(); + mdc.set_name("test-model"); + OpenAIPreprocessor::new(mdc).unwrap() + } + + #[tokio::test] + async fn preserves_url_and_uuid_only_slot_alignment() { + let messages = r#"[ + { + "role": "user", + "content": [ + {"type": "text", "text": "compare"}, + { + "type": "image_url", + "image_url": {"url": "https://example.com/first.png"}, + "uuid": "image-a" + }, + { + "type": "image_url", + "image_url": null, + "uuid": "image-b" + } + ] + } + ]"#; + let request = Request::from(messages, None, None, "test-model".to_string()); + + let (preprocessed, _, _) = make_preprocessor() + .preprocess_request(&request, None) + .await + .unwrap(); + + let images = &preprocessed.multi_modal_data.as_ref().unwrap()["image_url"]; + assert_eq!(images.len(), 2); + assert!(matches!(images[0], MultimodalData::Url(_))); + assert!(matches!( + &images[1], + MultimodalData::UuidOnly(value) if value == "image-b" + )); + assert_eq!( + preprocessed.multi_modal_uuids.as_ref().unwrap()["image_url"], + vec![Some("image-a".to_string()), Some("image-b".to_string())] + ); + } + + #[tokio::test] + async fn rejects_media_part_without_url_or_uuid() { + let messages = r#"[ + { + "role": "user", + "content": [ + {"type": "image_url", "image_url": null} + ] + } + ]"#; + let request = Request::from(messages, None, None, "test-model".to_string()); + + let error = make_preprocessor() + .preprocess_request(&request, None) + .await + .expect_err("media without a URL or UUID must be rejected"); + + let error_chain = format!("{error:#}"); + assert!( + error_chain.contains("neither `url` nor `uuid`"), + "unexpected error: {error_chain}" + ); + } + + #[tokio::test] + async fn rejects_empty_media_uuid() { + let messages = r#"[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + "uuid": "" + } + ] + } + ]"#; + let request = Request::from(messages, None, None, "test-model".to_string()); + + let error = make_preprocessor() + .preprocess_request(&request, None) + .await + .expect_err("an empty media UUID must be rejected"); + + let error_chain = format!("{error:#}"); + assert!( + error_chain.contains("uuid must be a non-empty string"), + "unexpected error: {error_chain}" + ); + } +} + mod context_length_validation { use dynamo_llm::model_card::ModelDeploymentCard; use dynamo_llm::preprocessor::OpenAIPreprocessor; diff --git a/tests/serve/multimodal_profiles/vllm.py b/tests/serve/multimodal_profiles/vllm.py index 8d061656c9f1..b54bcc15bf9b 100644 --- a/tests/serve/multimodal_profiles/vllm.py +++ b/tests/serve/multimodal_profiles/vllm.py @@ -15,6 +15,7 @@ make_image_payload, make_image_payload_b64, make_image_payload_cached_tokens, + make_image_payload_uuid_passthrough, make_video_payload, ) from tests.utils.payload_builder import chat_payload, chat_payload_default @@ -415,7 +416,14 @@ timeout_s=670, profiled_vram_gib=12.0, requested_vllm_kv_cache_bytes=922_354_000, - tests=[MmCase(payload=make_image_payload(["green"]))], + tests=[ + MmCase(payload=make_image_payload(["green"])), + MmCase( + suffix="uuid_passthrough", + payload=make_image_payload_uuid_passthrough(["green"]), + extra_script_args=["--mm-processor-cache-gb", "4"], + ), + ], ), }, extra_vllm_args=["--dtype", "bfloat16"], diff --git a/tests/utils/client.py b/tests/utils/client.py index 355c7f008de7..d4b472180202 100644 --- a/tests/utils/client.py +++ b/tests/utils/client.py @@ -39,8 +39,10 @@ def _sanitize_payload_for_logging(payload: Dict[str, Any]) -> Dict[str, Any]: if isinstance(content, list): for part in content: if isinstance(part, dict) and part.get("type") == "image_url": - image_url = part.get("image_url", {}) - if "url" in image_url: + image_url = part.get("image_url") + if isinstance(image_url, dict) and isinstance( + image_url.get("url"), str + ): image_url["url"] = _truncate_base64_url(image_url["url"]) return sanitized diff --git a/tests/utils/multimodal.py b/tests/utils/multimodal.py index 374785212eee..151827ecfa15 100644 --- a/tests/utils/multimodal.py +++ b/tests/utils/multimodal.py @@ -3,6 +3,7 @@ import base64 import os +from copy import deepcopy from dataclasses import dataclass, field from pathlib import Path from typing import Any, Optional, Type @@ -133,6 +134,95 @@ def make_image_payload_cached_tokens( ) +class UuidPassthroughChatPayload(ChatPayload): + """Send a URL+UUID cache fill followed by a UUID-only cache hit.""" + + def __init__( + self, + *, + expected_response: list[str], + image_uuid: str = "dynamo-mm-cache-image-1", + max_tokens: int = 100, + temperature: float = 0.0, + timeout: int = 60, + ): + object.__setattr__(self, "_uuid_initializing", True) + object.__setattr__(self, "_uuid_model", None) + object.__setattr__(self, "_uuid_image_uuid", image_uuid) + object.__setattr__(self, "_uuid_max_tokens", max_tokens) + object.__setattr__(self, "_uuid_temperature", temperature) + object.__setattr__(self, "_uuid_request_index", 0) + object.__setattr__(self, "_uuid_body_storage", None) + super().__init__( + body=None, + expected_response=expected_response, + expected_log=[], + repeat_count=2, + timeout=timeout, + ) + object.__setattr__(self, "_uuid_initializing", False) + + def with_model(self, model): + payload = deepcopy(self) + object.__setattr__(payload, "_uuid_model", model) + object.__setattr__(payload, "_uuid_request_index", 0) + return payload + + @property + def body(self) -> dict: # type: ignore[override] + request_index = self._uuid_request_index + object.__setattr__(self, "_uuid_request_index", request_index + 1) + + if request_index == 0: + image_url = {"url": MULTIMODAL_IMG_URL} + text = _MULTIMODAL_COLOR_PROMPT + else: + image_url = None + text = ( + "What colors are prominent in the same image? " + "Respond only with the colors." + ) + + body = { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": text}, + { + "type": "image_url", + "image_url": image_url, + "uuid": self._uuid_image_uuid, + }, + ], + } + ], + "max_tokens": self._uuid_max_tokens, + "temperature": self._uuid_temperature, + "stream": False, + } + if self._uuid_model is not None: + body["model"] = self._uuid_model + return body + + @body.setter + def body(self, value) -> None: + object.__setattr__(self, "_uuid_body_storage", value) + if not getattr(self, "_uuid_initializing", True): + object.__setattr__(self, "_uuid_request_index", 0) + + def final_validation(self) -> None: + assert self._uuid_request_index == self.repeat_count, ( + "UUID passthrough payload did not send the expected URL+UUID " + "and UUID-only request pair" + ) + + +def make_image_payload_uuid_passthrough(expected_response: list[str]) -> ChatPayload: + """Build the maintained vLLM cached multimodal UUID smoke payload.""" + return UuidPassthroughChatPayload(expected_response=expected_response) + + class Base64LazyChatPayload(ChatPayload): """ChatPayload variant that defers reading the multimodal test PNG until the first `.body` access. diff --git a/tests/utils/test_client.py b/tests/utils/test_client.py new file mode 100644 index 000000000000..ce1ee5d712bf --- /dev/null +++ b/tests/utils/test_client.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import pytest + +from tests.utils.client import _sanitize_payload_for_logging + +pytestmark = [ + pytest.mark.unit, + pytest.mark.pre_merge, + pytest.mark.gpu_0, +] + + +def test_sanitize_payload_preserves_uuid_only_image_part() -> None: + payload = { + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": None, + "uuid": "cached-image", + } + ], + } + ] + } + + assert _sanitize_payload_for_logging(payload) == payload From 5993ca60c851e894d638fb4631603974d8e610d3 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Wed, 15 Jul 2026 21:39:57 -0700 Subject: [PATCH 2/4] fix(multimodal): reject cache UUIDs outside vLLM Signed-off-by: Qi Wang --- .../dynamo/common/multimodal/cache_uuid.py | 18 +++++++ .../src/dynamo/frontend/sglang_processor.py | 7 ++- .../tests/test_sglang_processor_unit.py | 20 +++++++ .../request_handlers/llm/mm_disagg_utils.py | 5 ++ .../multimodal/encode_worker_handler.py | 4 ++ .../tests/test_sglang_decode_handler.py | 9 ++++ .../tests/test_sglang_multimodal_video.py | 12 +++++ .../request_handlers/aggregated_handler.py | 4 ++ .../trtllm/request_handlers/handler_base.py | 5 ++ .../trtllm/request_handlers/handlers.py | 10 ++++ .../test_trtllm_uuid_guard.py | 52 +++++++++++++++++++ .../trtllm/tests/test_trtllm_handler_base.py | 12 +++++ docs/features/multimodal/multimodal-vllm.md | 2 + .../trtllm/mm_router_worker/handler.py | 5 ++ 14 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 components/src/dynamo/common/multimodal/cache_uuid.py create mode 100644 components/src/dynamo/trtllm/tests/request_handlers/test_trtllm_uuid_guard.py diff --git a/components/src/dynamo/common/multimodal/cache_uuid.py b/components/src/dynamo/common/multimodal/cache_uuid.py new file mode 100644 index 000000000000..fa988721bdc5 --- /dev/null +++ b/components/src/dynamo/common/multimodal/cache_uuid.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Backend capability guard for client-provided multimodal cache UUIDs.""" + + +def reject_unsupported_multimodal_uuids( + multi_modal_uuids: object, + *, + backend: str, +) -> None: + if not multi_modal_uuids: + return + + raise ValueError( + "Multimodal UUID caching is currently supported only by the vLLM " + f"backend; {backend} does not support the `uuid` field" + ) diff --git a/components/src/dynamo/frontend/sglang_processor.py b/components/src/dynamo/frontend/sglang_processor.py index 91cc9901ae03..1b7d525b86ca 100644 --- a/components/src/dynamo/frontend/sglang_processor.py +++ b/components/src/dynamo/frontend/sglang_processor.py @@ -20,6 +20,7 @@ from sglang.srt.utils.hf_transformers_utils import get_tokenizer from dynamo._internal import ModelDeploymentCard +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.frontend.frontend_args import FrontendConfig from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine from dynamo.llm.exceptions import InvalidArgument, Unknown @@ -325,10 +326,12 @@ def _build_dynamo_preproc( # Forward multimodal URLs so the backend handler can load the media. mm_data, mm_uuids = extract_mm_urls(request.get("messages", [])) + try: + reject_unsupported_multimodal_uuids(mm_uuids, backend="SGLang") + except ValueError as exc: + raise PreprocessError(str(exc)) from exc if mm_data: preproc["multi_modal_data"] = mm_data - if mm_uuids: - preproc["multi_modal_uuids"] = mm_uuids nvext = request.get("nvext") or {} nvext_passthrough = { diff --git a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py index 8daa97adbf23..2e959eb316e2 100644 --- a/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_sglang_processor_unit.py @@ -102,6 +102,26 @@ def test_defaults(self): assert sampling["repetition_penalty"] == 1.0 assert sampling["seed"] is None + def test_rejects_multimodal_cache_uuid(self): + request = { + "model": "test", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png"}, + "uuid": "cached-image", + } + ], + } + ], + } + + with pytest.raises(PreprocessError, match="supported only by the vLLM backend"): + _build_dynamo_preproc(request, [1], "test", None) + def test_top_k_zero_maps_to_negative_one(self): """SGLang uses -1 for disabled top_k, OpenAI uses 0.""" result = _build_dynamo_preproc( diff --git a/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py b/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py index 71e279d4cac4..e0ce5b8bc039 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py +++ b/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py @@ -9,6 +9,8 @@ import logging from typing import Any, Dict, Optional +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids + logger = logging.getLogger(__name__) IMAGE_URL_KEY = "image_url" @@ -51,6 +53,9 @@ def _raw_multimodal_content_types(request: Dict[str, Any]) -> set[str]: def raise_if_unextracted_multimodal(request: Dict[str, Any]) -> None: """Reject raw multimodal messages that were not extracted by the frontend.""" + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="SGLang" + ) mm_data = _multi_modal_data(request) raw_types = _raw_multimodal_content_types(request) if not (mm_data or raw_types): diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py index cbe86a8aac10..a9a02479e0a6 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py @@ -25,6 +25,7 @@ MultimodalEmbeddingCacheManager, ) from dynamo.common.multimodal import EMBEDDING_SENDER_FACTORIES +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.common.utils import nvtx_utils as _nvtx from dynamo.llm import MultimodalEmbeddingCachePublisher from dynamo.sglang.args import Config @@ -435,6 +436,9 @@ def _extract_media_urls( Returns: Tuple of (image_urls, video_urls) lists. """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="SGLang" + ) mm_data = request.get("multi_modal_data") if not mm_data: raise ValueError("multi_modal_data is required for the encode worker.") diff --git a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py index 5ad4aefc11b2..6e9424d2f5db 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py +++ b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py @@ -318,6 +318,15 @@ def test_raises_for_image_url(self, request_factory): def test_text_only_request_bypasses_guard(self): raise_if_unextracted_multimodal({"token_ids": [10, 20, 30]}) + def test_rejects_multimodal_cache_uuid(self): + with pytest.raises(ValueError, match="supported only by the vLLM backend"): + raise_if_unextracted_multimodal( + { + "token_ids": [10, 20, 30], + "multi_modal_uuids": {"image_url": ["cached-image"]}, + } + ) + def test_build_logprob_kwargs_allows_chosen_token_logprobs(monkeypatch): monkeypatch.delenv("DYN_SGL_ALLOW_TOP_LOGPROBS", raising=False) diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index e8393824ca80..42d23f0f6736 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -65,6 +65,18 @@ def test_extract_media_urls_supports_mixed_image_and_video(): assert video_urls == ["https://example.com/clip.mp4"] +def test_extract_media_urls_rejects_multimodal_cache_uuid(): + handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler) + + with pytest.raises(ValueError, match="supported only by the vLLM backend"): + handler._extract_media_urls( + { + "multi_modal_data": {"image_url": [{"UuidOnly": "cached-image"}]}, + "multi_modal_uuids": {"image_url": ["cached-image"]}, + } + ) + + @pytest.mark.asyncio async def test_build_mm_items_routes_video_to_video_data(): embeddings = torch.arange(24, dtype=torch.float16).reshape(6, 4) diff --git a/components/src/dynamo/trtllm/request_handlers/aggregated_handler.py b/components/src/dynamo/trtllm/request_handlers/aggregated_handler.py index 2e66fdb47070..30d39bd4367c 100644 --- a/components/src/dynamo/trtllm/request_handlers/aggregated_handler.py +++ b/components/src/dynamo/trtllm/request_handlers/aggregated_handler.py @@ -13,6 +13,7 @@ from dynamo.common.memory.multimodal_embedding_cache_manager import ( MultimodalEmbeddingCacheManager, ) +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.trtllm.multimodal.embedding_fetcher import fetch_embeddings_from_encoder from dynamo.trtllm.request_handlers.handler_base import ( HandlerBase, @@ -40,6 +41,9 @@ async def generate( self, request: dict, context: Context ) -> AsyncGenerator[dict, None]: """Generate response, optionally using remote encoder for multimodal.""" + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) logging.debug(f"AggregatedHandler Request ID: {context.id()}") embeddings: Optional[Union[torch.Tensor, dict]] = None diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index 5996eb6c46d9..cdde23ba2be1 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -37,6 +37,7 @@ from dynamo.common.backend import logprobs as _shared_logprobs from dynamo.common.backend.engine import is_generation_stage from dynamo.common.constants import DisaggregationMode as CommonDisaggregationMode +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.common.utils.structural_tag import serialize_structural_tag from dynamo.health_check import HEALTH_CHECK_KEY from dynamo.llm.exceptions import EngineShutdown @@ -789,6 +790,10 @@ async def _prepare_input_for_generation( Returns: Processed input for TRT-LLM (dict with prompt/token_ids, or raw token_ids) """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) + # DECODE mode: Use prefill metadata to skip re-processing multimodal content # Per TRT-LLM team: DECODE never needs to reload images - KV cache has the context has_prefill_metadata = epd_metadata and ( diff --git a/components/src/dynamo/trtllm/request_handlers/handlers.py b/components/src/dynamo/trtllm/request_handlers/handlers.py index d9c9a247ba4c..247cc453ce93 100644 --- a/components/src/dynamo/trtllm/request_handlers/handlers.py +++ b/components/src/dynamo/trtllm/request_handlers/handlers.py @@ -9,6 +9,7 @@ from dynamo.common.memory.multimodal_embedding_cache_manager import ( MultimodalEmbeddingCacheManager, ) +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.runtime.logging import configure_dynamo_logging from dynamo.trtllm.encode_helper import EncodeHelper from dynamo.trtllm.multimodal.embedding_fetcher import fetch_embeddings_from_encoder @@ -69,6 +70,9 @@ def __init__(self, config: RequestHandlerConfig): async def generate( self, request: dict, context: Context ) -> AsyncGenerator[dict, None]: + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) logging.debug(f"New Request ID: {context.id()}") if self.multimodal_processor is None: logging.error("encode handler: no multimodal_processor configured") @@ -136,6 +140,9 @@ async def generate( Prefill worker: process prompt and return disaggregated_params. Frontend routes to decode workers automatically. """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) logging.debug(f"Prefill Request ID: {context.id()}") request_token_ids = request.get("token_ids") logging.debug( @@ -220,6 +227,9 @@ async def generate( Decode worker: generate tokens using disaggregated_params from prefill. If disaggregated_params is present, prefill was done. Otherwise generate normally. """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) logging.debug(f"Decode Request ID: {context.id()}") disaggregated_params = request.get("disaggregated_params") diff --git a/components/src/dynamo/trtllm/tests/request_handlers/test_trtllm_uuid_guard.py b/components/src/dynamo/trtllm/tests/request_handlers/test_trtllm_uuid_guard.py new file mode 100644 index 000000000000..be4fb2a5c6dc --- /dev/null +++ b/components/src/dynamo/trtllm/tests/request_handlers/test_trtllm_uuid_guard.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for rejecting vLLM-only multimodal cache UUIDs.""" + +import pytest +import torch + +if not torch.cuda.is_available(): + pytest.skip( + "Skipping because TensorRT-LLM imports require a CUDA-capable GPU.", + allow_module_level=True, + ) + +from dynamo.trtllm.request_handlers.aggregated_handler import AggregatedHandler +from dynamo.trtllm.request_handlers.handler_base import HandlerBase +from dynamo.trtllm.request_handlers.handlers import ( + DecodeHandler, + EncodeHandler, + PrefillHandler, +) +from dynamo.trtllm.tests.request_handlers.utils import create_mock_context +from dynamo.trtllm.tests.utils import create_mock_request_handler_config + +pytestmark = [ + pytest.mark.pre_merge, + pytest.mark.unit, + pytest.mark.trtllm, + pytest.mark.gpu_0, +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("handler_type", "disaggregation_mode"), + [ + (EncodeHandler, "encode"), + (PrefillHandler, "prefill"), + (DecodeHandler, "decode"), + (AggregatedHandler, "prefill_and_decode"), + ], +) +async def test_handler_rejects_multimodal_cache_uuid( + handler_type: type[HandlerBase], disaggregation_mode: str +) -> None: + config = create_mock_request_handler_config(disaggregation_mode=disaggregation_mode) + handler = handler_type(config) + request = {"multi_modal_uuids": {"image_url": ["cached-image"]}} + + with pytest.raises(ValueError, match="supported only by the vLLM backend"): + async for _ in handler.generate(request, create_mock_context()): + pass diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index 030d5a0d0dcd..8cc5578ae330 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -625,6 +625,18 @@ async def test_text_only_request_falls_back_to_token_ids(self): result = await self._prepare(handler, {"token_ids": [10, 20, 30]}) assert result == [10, 20, 30] + @pytest.mark.asyncio + async def test_rejects_multimodal_cache_uuid(self): + handler = self._make_handler(multimodal_processor=MagicMock()) + request = { + "token_ids": [1, 2, 3], + "multi_modal_data": {"image_url": [{"UuidOnly": "cached-image"}]}, + "multi_modal_uuids": {"image_url": ["cached-image"]}, + } + + with pytest.raises(ValueError, match="supported only by the vLLM backend"): + await self._prepare(handler, request) + @pytest.mark.asyncio async def test_decode_with_prefill_metadata_bypasses_guard(self): handler = self._make_handler(multimodal_processor=None) diff --git a/docs/features/multimodal/multimodal-vllm.md b/docs/features/multimodal/multimodal-vllm.md index 7cd758e9d713..ec252dadd268 100644 --- a/docs/features/multimodal/multimodal-vllm.md +++ b/docs/features/multimodal/multimodal-vllm.md @@ -107,6 +107,8 @@ curl http://localhost:8000/v1/chat/completions \ vLLM can cache processed image, video, and audio inputs under a client-provided opaque UUID. This extension is specific to vLLM; it is not part of the OpenAI chat completions schema and is not supported by Dynamo's other backends. +Dynamo's SGLang and TensorRT-LLM backends reject requests containing this +`uuid` field rather than silently ignoring unsupported cache semantics. To enable the cache, pass a nonzero `--mm-processor-cache-gb` value to the vLLM worker. Populate an entry by adding `uuid` beside the media field: diff --git a/examples/backends/trtllm/mm_router_worker/handler.py b/examples/backends/trtllm/mm_router_worker/handler.py index 078d223c91c0..2ff07b1c8a69 100644 --- a/examples/backends/trtllm/mm_router_worker/handler.py +++ b/examples/backends/trtllm/mm_router_worker/handler.py @@ -8,6 +8,7 @@ import logging from typing import Any, AsyncGenerator +from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids from dynamo.llm import KvRouter from dynamo.runtime.logging import configure_dynamo_logging @@ -68,6 +69,10 @@ async def generate(self, request: dict) -> AsyncGenerator[dict, None]: Yields: Response chunks from the downstream TRT-LLM worker via KvRouter """ + reject_unsupported_multimodal_uuids( + request.get("multi_modal_uuids"), backend="TensorRT-LLM" + ) + # Extract messages from extra_args (set by Frontend preprocessor) messages = request.get("extra_args", {}).get("messages", []) image_urls = extract_image_urls(messages) From 534436df822887e5e804fa80f7fa4d57bde1c540 Mon Sep 17 00:00:00 2001 From: Qi Wang Date: Thu, 16 Jul 2026 00:09:24 -0700 Subject: [PATCH 3/4] fix(multimodal): limit cache UUIDs to images Signed-off-by: Qi Wang --- Cargo.lock | 3 +- Cargo.toml | 4 +- .../dynamo/common/multimodal/audio_loader.py | 20 +-- .../dynamo/common/multimodal/cache_uuid.py | 2 +- .../dynamo/common/multimodal/video_loader.py | 25 +-- .../tests/multimodal/test_audio_loader.py | 24 --- .../tests/multimodal/test_video_loader.py | 30 ---- .../frontend/tests/test_multimodal_utils.py | 35 +++++ .../tests/test_vllm_processor_unit.py | 26 ++++ components/src/dynamo/frontend/utils.py | 38 +++-- .../src/dynamo/frontend/vllm_processor.py | 43 ++++-- .../multimodal_utils/request_processor.py | 104 +++---------- .../test_vllm_request_processor.py | 146 +++--------------- docs/features/multimodal/multimodal-vllm.md | 26 ++-- lib/bindings/kvbm/Cargo.lock | 3 +- lib/bindings/kvbm/Cargo.toml | 2 +- lib/bindings/python/Cargo.lock | 3 +- lib/bindings/python/Cargo.toml | 2 +- lib/llm/src/preprocessor.rs | 70 ++++++--- lib/llm/src/preprocessor/media/loader.rs | 18 +-- lib/llm/src/protocols/anthropic/types.rs | 11 +- lib/llm/src/protocols/common/preprocessor.rs | 6 +- lib/llm/src/protocols/openai/responses/mod.rs | 4 +- lib/llm/tests/preprocessor.rs | 64 ++++++++ 24 files changed, 326 insertions(+), 383 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cbdca86d55d..2fc3587b3c8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2759,7 +2759,7 @@ dependencies = [ [[package]] name = "dynamo-protocols" version = "2.0.2" -source = "git+https://github.com/furionw/frontend-crates.git?rev=31cd250192dbca6b3005bec8170e6f95d2bc0abf#31cd250192dbca6b3005bec8170e6f95d2bc0abf" +source = "git+https://github.com/furionw/frontend-crates.git?rev=522a09c319772e3e8c4250f9afd917e7a460f2cc#522a09c319772e3e8c4250f9afd917e7a460f2cc" dependencies = [ "async-openai", "derive_builder", @@ -2769,6 +2769,7 @@ dependencies = [ "thiserror 2.0.18", "tracing", "url", + "uuid", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 66aae9bae2a1..306ec2d0b77d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -53,7 +53,7 @@ dynamo-kv-hashing = { path = "lib/kv-hashing", version = "1.3.0" } dynamo-memory = { path = "lib/memory", version = "1.3.0" } dynamo-mocker = { path = "lib/mocker", version = "1.3.0" } dynamo-kv-router = { path = "lib/kv-router", version = "1.3.0", features = ["metrics", "runtime-protocols"] } -dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" } +dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "522a09c319772e3e8c4250f9afd917e7a460f2cc" } dynamo-parsers = { version = "5.0.0" } # Published on crates.io. To see all available versions: # web: https://crates.io/crates/dynamo-parsers-v2/versions @@ -182,7 +182,7 @@ rustls-pemfile = { version = "2" } [patch.crates-io] # Temporary until ai-dynamo/frontend-crates#119 is released. -dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" } +dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "522a09c319772e3e8c4250f9afd917e7a460f2cc" } [profile.dev.package] insta.opt-level = 3 diff --git a/components/src/dynamo/common/multimodal/audio_loader.py b/components/src/dynamo/common/multimodal/audio_loader.py index 559755360d7b..c0095f767a3f 100644 --- a/components/src/dynamo/common/multimodal/audio_loader.py +++ b/components/src/dynamo/common/multimodal/audio_loader.py @@ -18,7 +18,6 @@ # Constants for multimodal data variants URL_VARIANT_KEY: Final = "Url" DECODED_VARIANT_KEY: Final = "Decoded" -UUID_ONLY_VARIANT_KEY: Final = "UuidOnly" def _create_nixl_connector() -> Any: @@ -171,7 +170,7 @@ async def _load_decoded_audio( async def load_audio_batch( self, audio_mm_items: List[Dict[str, Any]], - ) -> List[tuple[np.ndarray, float] | None]: + ) -> List[tuple[np.ndarray, float]]: """Load a batch of audio files from multimodal data items. Supports two paths: @@ -183,42 +182,31 @@ async def load_audio_batch( List of (waveform, sample_rate) tuples. """ audio_futures: List[Awaitable[tuple[np.ndarray, float]]] = [] - slot_to_future_idx: list[int | None] = [] for idx, item in enumerate(audio_mm_items): if isinstance(item, dict) and URL_VARIANT_KEY in item: url = item[URL_VARIANT_KEY] - slot_to_future_idx.append(len(audio_futures)) audio_futures.append(self.load_audio(url)) logger.debug("Preparing to load audio from URL: %s...", url[:80]) elif isinstance(item, dict) and DECODED_VARIANT_KEY in item: if self._enable_frontend_decoding: metadata = item[DECODED_VARIANT_KEY] - slot_to_future_idx.append(len(audio_futures)) audio_futures.append(self._load_decoded_audio(metadata)) else: raise ValueError( "Received decoded audio data but enable_frontend_decoding=False. " "Enable frontend decoding to transfer decoded audio via NIXL." ) - elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item: - slot_to_future_idx.append(None) else: raise ValueError( f"Invalid audio multimodal item at index {idx}. " - "Expected dict with 'Url', 'Decoded', or 'UuidOnly' key." + "Expected dict with 'Url' or 'Decoded' key." ) results = await asyncio.gather(*audio_futures, return_exceptions=True) - loaded_audio: list[tuple[np.ndarray, float] | None] = [] + loaded_audio: list[tuple[np.ndarray, float]] = [] collective_exceptions: list[str] = [] - for media_item, future_idx in zip( - audio_mm_items, slot_to_future_idx, strict=True - ): - if future_idx is None: - loaded_audio.append(None) - continue - result = results[future_idx] + for media_item, result in zip(audio_mm_items, results, strict=True): if isinstance(result, BaseException): if isinstance(result, asyncio.CancelledError): raise result diff --git a/components/src/dynamo/common/multimodal/cache_uuid.py b/components/src/dynamo/common/multimodal/cache_uuid.py index fa988721bdc5..ae2c9bef9e03 100644 --- a/components/src/dynamo/common/multimodal/cache_uuid.py +++ b/components/src/dynamo/common/multimodal/cache_uuid.py @@ -13,6 +13,6 @@ def reject_unsupported_multimodal_uuids( return raise ValueError( - "Multimodal UUID caching is currently supported only by the vLLM " + "Image UUID caching is currently supported only by the vLLM " f"backend; {backend} does not support the `uuid` field" ) diff --git a/components/src/dynamo/common/multimodal/video_loader.py b/components/src/dynamo/common/multimodal/video_loader.py index e8247b081a9f..a5b18590a1cb 100644 --- a/components/src/dynamo/common/multimodal/video_loader.py +++ b/components/src/dynamo/common/multimodal/video_loader.py @@ -30,7 +30,6 @@ URL_VARIANT_KEY: Final = "Url" DECODED_VARIANT_KEY: Final = "Decoded" -UUID_ONLY_VARIANT_KEY: Final = "UuidOnly" def _create_nixl_connector() -> Any: @@ -162,44 +161,28 @@ async def _load_decoded_video( async def load_video_batch( self, video_mm_items: List[Dict[str, Any]], - ) -> List[tuple[np.ndarray, Dict[str, Any]] | None]: + ) -> List[tuple[np.ndarray, Dict[str, Any]]]: video_futures: List[Awaitable[tuple[np.ndarray, Dict[str, Any]]]] = [] - slot_to_future_idx: list[int | None] = [] - for idx, item in enumerate(video_mm_items): + for item in video_mm_items: if isinstance(item, dict) and URL_VARIANT_KEY in item: url = item[URL_VARIANT_KEY] - slot_to_future_idx.append(len(video_futures)) video_futures.append(self.load_video(url)) logger.debug("Preparing to load video from URL: %s...", url[:80]) elif isinstance(item, dict) and DECODED_VARIANT_KEY in item: if self._enable_frontend_decoding: metadata = item[DECODED_VARIANT_KEY] - slot_to_future_idx.append(len(video_futures)) video_futures.append(self._load_decoded_video(metadata)) else: raise ValueError( "Received decoded video data but enable_frontend_decoding=False. " "Enable frontend decoding to transfer decoded video frames via NIXL." ) - elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item: - slot_to_future_idx.append(None) - else: - raise ValueError( - f"Invalid video multimodal item at index {idx}. " - "Expected dict with 'Url', 'Decoded', or 'UuidOnly' key." - ) results = await asyncio.gather(*video_futures, return_exceptions=True) - loaded_videos: list[tuple[np.ndarray, Dict[str, Any]] | None] = [] + loaded_videos: list[tuple[np.ndarray, Dict[str, Any]]] = [] collective_exceptions: list[str] = [] - for media_item, future_idx in zip( - video_mm_items, slot_to_future_idx, strict=True - ): - if future_idx is None: - loaded_videos.append(None) - continue - result = results[future_idx] + for media_item, result in zip(video_mm_items, results): if isinstance(result, BaseException): if isinstance(result, asyncio.CancelledError): raise result diff --git a/components/src/dynamo/common/tests/multimodal/test_audio_loader.py b/components/src/dynamo/common/tests/multimodal/test_audio_loader.py index 8e6ec3efb67e..5a6eb4a188cc 100644 --- a/components/src/dynamo/common/tests/multimodal/test_audio_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_audio_loader.py @@ -136,27 +136,3 @@ async def test_load_audio_batch_reads_decoded_variant(monkeypatch): decoded_item, return_metadata=True, ) - - -@pytest.mark.asyncio -async def test_uuid_only_audio_slots_preserve_batch_alignment(): - loader = AudioLoader() - first = (np.zeros(8, dtype=np.float32), 16000.0) - second = (np.ones(8, dtype=np.float32), 44100.0) - loader.load_audio = AsyncMock( # type: ignore[method-assign] - side_effect=[first, second] - ) - - audios = await loader.load_audio_batch( - [ - {"Url": "https://example.com/first.wav"}, - {"UuidOnly": "cached-audio"}, - {"Url": "https://example.com/second.wav"}, - ] - ) - - assert audios[0] is not None - assert audios[1] is None - assert audios[2] is not None - np.testing.assert_array_equal(audios[0][0], first[0]) - np.testing.assert_array_equal(audios[2][0], second[0]) diff --git a/components/src/dynamo/common/tests/multimodal/test_video_loader.py b/components/src/dynamo/common/tests/multimodal/test_video_loader.py index fe9a353273ff..7158cb80a5bc 100644 --- a/components/src/dynamo/common/tests/multimodal/test_video_loader.py +++ b/components/src/dynamo/common/tests/multimodal/test_video_loader.py @@ -109,33 +109,3 @@ async def test_load_video_batch_reads_decoded_variant_with_metadata(monkeypatch) decoded_item, return_metadata=True, ) - - -@pytest.mark.asyncio -async def test_uuid_only_video_slots_preserve_batch_alignment(): - loader = VideoLoader() - first = ( - np.zeros((1, 2, 2, 3), dtype=np.uint8), - {"fps": 2.0, "frames_indices": [0], "total_num_frames": 1}, - ) - second = ( - np.ones((1, 2, 2, 3), dtype=np.uint8), - {"fps": 2.0, "frames_indices": [0], "total_num_frames": 1}, - ) - loader.load_video = AsyncMock( # type: ignore[method-assign] - side_effect=[first, second] - ) - - videos = await loader.load_video_batch( - [ - {"Url": "https://example.com/first.mp4"}, - {"UuidOnly": "cached-video"}, - {"Url": "https://example.com/second.mp4"}, - ] - ) - - assert videos[0] is not None - assert videos[1] is None - assert videos[2] is not None - np.testing.assert_array_equal(videos[0][0], first[0]) - np.testing.assert_array_equal(videos[2][0], second[0]) diff --git a/components/src/dynamo/frontend/tests/test_multimodal_utils.py b/components/src/dynamo/frontend/tests/test_multimodal_utils.py index 9dda4f0011a0..893ead26925e 100644 --- a/components/src/dynamo/frontend/tests/test_multimodal_utils.py +++ b/components/src/dynamo/frontend/tests/test_multimodal_utils.py @@ -212,6 +212,41 @@ def test_extracts_url_and_uuid_only_slots_with_alignment(): ) +def test_accepts_deprecated_nested_image_uuid() -> None: + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/a.png", + "uuid": "92b888ad-e64a-478f-b688-5091e16544e3", + }, + } + ], + } + ] + + assert extract_mm_urls(messages) == ( + {"image_url": [{"Url": "https://example.com/a.png"}]}, + {"image_url": ["92b888ad-e64a-478f-b688-5091e16544e3"]}, + ) + + +@pytest.mark.parametrize("part_type", ["audio_url", "video_url"]) +@pytest.mark.parametrize("nested", [False, True]) +def test_rejects_audio_video_cache_uuids(part_type: str, nested: bool) -> None: + media = {"url": "https://example.com/media", "uuid": "cached-media"} + part = {"type": part_type, part_type: media} + if not nested: + media.pop("uuid") + part["uuid"] = "cached-media" + + with pytest.raises(ValueError, match="supported only for image_url.*vLLM"): + extract_mm_urls([{"role": "user", "content": [part]}]) + + @pytest.mark.parametrize( "part", [ diff --git a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py index 60eee67bdfac..d8ceb3ddad4a 100644 --- a/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py +++ b/components/src/dynamo/frontend/tests/test_vllm_processor_unit.py @@ -568,6 +568,32 @@ async def process_for_engine_async(self, prompt, arrival_time): } +def test_normalize_vllm_image_parts_lifts_deprecated_nested_uuid( + vllm_processor_module, +) -> None: + messages = [ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/image.png", + "detail": None, + "uuid": "92b888ad-e64a-478f-b688-5091e16544e3", + }, + } + ], + } + ] + + vllm_processor_module._normalize_vllm_image_parts(messages) + + part = messages[0]["content"][0] + assert part["uuid"] == "92b888ad-e64a-478f-b688-5091e16544e3" + assert part["image_url"]["detail"] == "auto" + + class _FakeOutputProcessor: def __init__(self): self.request_states = {} diff --git a/components/src/dynamo/frontend/utils.py b/components/src/dynamo/frontend/utils.py index 607d4df3213b..d49ac1bb1dab 100644 --- a/components/src/dynamo/frontend/utils.py +++ b/components/src/dynamo/frontend/utils.py @@ -100,11 +100,13 @@ def __init__(self, message: str): def extract_mm_urls( messages: list[dict[str, Any]], ) -> tuple[dict[str, list[dict[str, str]]] | None, dict[str, list[str | None]] | None,]: - """Extract aligned multimodal data and cache UUIDs from chat messages. + """Extract media and vLLM image processor-cache UUIDs from chat messages. - URL-backed parts become ``Url`` variants. Parts with no URL and an opaque - top-level ``uuid`` become ``UuidOnly`` variants for vLLM's multimodal - processor cache. UUID lists preserve modality-local slot order:: + URL-backed parts become ``Url`` variants. Image parts with no URL and an + opaque ``uuid`` become ``UuidOnly`` variants for vLLM's multimodal + processor cache. The deprecated nested image UUID remains accepted, while + cache UUIDs on audio and video are rejected. Image UUID lists preserve + slot order:: ({"image_url": [{"Url": "https://..."}, {"UuidOnly": "image-1"}]}, {"image_url": ["image-1", "image-1"]}) @@ -129,13 +131,24 @@ def extract_mm_urls( if part_type not in _MEDIA_CONTENT_TYPES: continue - uuid_value = part.get("uuid") - if uuid_value is not None and ( - not isinstance(uuid_value, str) or not uuid_value - ): - raise ValueError(f"{part_type} uuid must be a non-empty string") - media_value = part.get(part_type) + top_level_uuid = part.get("uuid") + nested_uuid = ( + media_value.get("uuid") if isinstance(media_value, dict) else None + ) + for uuid_value in (top_level_uuid, nested_uuid): + if uuid_value is not None and ( + not isinstance(uuid_value, str) or not uuid_value + ): + raise ValueError(f"{part_type} uuid must be a non-empty string") + + uuid_value = top_level_uuid if top_level_uuid is not None else nested_uuid + if uuid_value is not None and part_type != "image_url": + raise ValueError( + "multimodal cache UUIDs are currently supported only for " + "image_url parts with vLLM" + ) + url = media_value.get("url") if isinstance(media_value, dict) else None if isinstance(url, str) and url: mm_data.setdefault(part_type, []).append({"Url": url}) @@ -146,8 +159,9 @@ def extract_mm_urls( f"{part_type} part must contain a non-empty URL or uuid" ) - mm_uuids.setdefault(part_type, []).append(uuid_value) - any_uuid |= uuid_value is not None + if part_type == "image_url": + mm_uuids.setdefault(part_type, []).append(uuid_value) + any_uuid |= uuid_value is not None return mm_data or None, mm_uuids if any_uuid else None diff --git a/components/src/dynamo/frontend/vllm_processor.py b/components/src/dynamo/frontend/vllm_processor.py index 0c52e27d1a4b..3b126de6f79a 100644 --- a/components/src/dynamo/frontend/vllm_processor.py +++ b/components/src/dynamo/frontend/vllm_processor.py @@ -208,6 +208,28 @@ async def _build_engine_inputs( return await renderer.process_for_engine_async(prompt_inputs, time.time()) +def _normalize_vllm_image_parts(messages: list[Any]) -> None: + """Normalize image parts before vLLM validates and renders them.""" + for msg in messages: + if not isinstance(msg, dict): + continue + content = msg.get("content") + if not isinstance(content, list): + continue + for part in content: + if not isinstance(part, dict) or part.get("type") != "image_url": + continue + image_url = part.get("image_url") + if not isinstance(image_url, dict): + continue + if image_url.get("detail") is None: + image_url["detail"] = "auto" + # Older Dynamo clients nested a UUID inside image_url. Lift it into + # vLLM's canonical content-part field before the vLLM parser sees it. + if part.get("uuid") is None and image_url.get("uuid") is not None: + part["uuid"] = image_url["uuid"] + + class VllmProcessor: def __init__( self, @@ -407,22 +429,12 @@ async def _generator_inner( ) -> AsyncGenerator[dict[str, Any], None]: request_id = random_uuid() - # vLLM's Pydantic model requires image_url.detail to be 'auto'/'low'/'high'. - # The Rust HTTP layer accepts None/missing, so normalize before validation. messages = request.get("messages") or [] - for msg in messages: - if not isinstance(msg, dict): - continue - content = msg.get("content") - if not isinstance(content, list): - continue - for part in content: - if not isinstance(part, dict): - continue - if part.get("type") == "image_url": - img_url = part.get("image_url") - if isinstance(img_url, dict) and img_url.get("detail") is None: - img_url["detail"] = "auto" + _normalize_vllm_image_parts(messages) + # Validate cache-UUID modality support before vLLM downloads or + # processes media. Dynamo currently exposes vLLM cache UUIDs for + # images only. + mm_data, mm_uuids = extract_mm_urls(messages) # Images are fetched by vLLM's renderer via DynamoMediaConnector, # which wraps our ImageLoader (LRU cache + in-flight dedup). @@ -568,7 +580,6 @@ async def _generator_inner( # Parse user cache identities before building routing metadata. Opaque # UUIDs deliberately suppress multimodal exact routing while retaining # vLLM's feature metadata for backend cache reuse and tensor transfer. - mm_data, mm_uuids = extract_mm_urls(request.get("messages") or []) if mm_uuids: dynamo_preproc["multi_modal_uuids"] = mm_uuids diff --git a/components/src/dynamo/vllm/multimodal_utils/request_processor.py b/components/src/dynamo/vllm/multimodal_utils/request_processor.py index b66f1f865542..e2a9d3b5541c 100644 --- a/components/src/dynamo/vllm/multimodal_utils/request_processor.py +++ b/components/src/dynamo/vllm/multimodal_utils/request_processor.py @@ -47,7 +47,6 @@ VIDEO_URL_KEY = "video_url" AUDIO_URL_KEY = "audio_url" URL_VARIANT_KEY = "Url" -UUID_ONLY_VARIANT_KEY = "UuidOnly" def pad_mm_hashes_to_64(mm_hashes: list[str]) -> list[str]: @@ -104,50 +103,31 @@ def _build_user_mm_uuids( raw_uuids: Any, use_unified_vision_chunk: bool, ) -> Optional[dict[str, list[str | None]]]: - """Normalize user cache identities without changing opaque values.""" + """Normalize vLLM image cache identities without changing opaque values.""" if not isinstance(raw_uuids, dict): return None - modality_names = { - IMAGE_URL_KEY: "image", - VIDEO_URL_KEY: "video", - AUDIO_URL_KEY: "audio", - } - normalized: dict[str, list[str | None]] = {} for modality, values in raw_uuids.items(): - if not isinstance(values, list): + if modality == IMAGE_URL_KEY: continue - backend_modality = modality_names.get(str(modality), str(modality)) - backend_modality = _normalize_forwarded_mm_modality( - backend_modality, - use_unified_vision_chunk, + has_uuid = ( + any(value is not None for value in values) + if isinstance(values, list) + else values is not None ) - normalized[backend_modality] = list(values) - return normalized or None - - -def _append_video_audio_uuid_slots( - mm_uuids: dict[str, list[str | None]], - request: dict[str, Any], - mm_processor_kwargs: Optional[dict[str, Any]], -) -> None: - """Align UUIDs for audio tracks that vLLM extracts from video inputs.""" - if not mm_processor_kwargs or not mm_processor_kwargs.get( - "use_audio_in_video", False - ): - return - - mm_data = request.get("multi_modal_data") - if not isinstance(mm_data, dict): - return - video_items = mm_data.get(VIDEO_URL_KEY) - if not isinstance(video_items, list) or not video_items: - return - - audio_items = mm_data.get(AUDIO_URL_KEY) - standalone_audio_count = len(audio_items) if isinstance(audio_items, list) else 0 - audio_uuids = mm_uuids.setdefault("audio", [None] * standalone_audio_count) - audio_uuids.extend([None] * len(video_items)) + if has_uuid: + raise ValueError( + "multimodal cache UUIDs are currently supported only for images" + ) + + image_uuids = raw_uuids.get(IMAGE_URL_KEY) + if not isinstance(image_uuids, list): + return None + backend_modality = _normalize_forwarded_mm_modality( + "image", + use_unified_vision_chunk, + ) + return {backend_modality: list(image_uuids)} def _get_modality_extra_values( @@ -191,22 +171,6 @@ def _placeholder_range_from_extra_arg(value: Any) -> PlaceholderRange: return PlaceholderRange(offset=offset, length=length) -def _normalize_optional_media_batch( - values: list[Any | None], - modality: str, -) -> Any: - """Shape UUID-only video/audio batches for vLLM's modality parsers.""" - if all(value is None for value in values): - return None - if any(value is None for value in values): - raise ValueError( - f"Mixed URL/decoded and UUID-only {modality} inputs are not " - "supported by vLLM; provide media for every item or use only " - "cached UUIDs for this modality." - ) - return values[0] if len(values) == 1 else values - - def compute_mm_uuids( multi_modal_data: Optional[dict[str, Any]], ) -> Optional[dict[str, list[str]]]: @@ -437,42 +401,28 @@ async def extract_multimodal_data( if video_items: videos = await self.video_loader.load_video_batch(video_items) if videos: - vllm_mm_data["video"] = _normalize_optional_media_batch( - videos, - "video", - ) + vllm_mm_data["video"] = videos[0] if len(videos) == 1 else videos audio_items = mm_map.get(AUDIO_URL_KEY, []) if audio_items: audios = await self.audio_loader.load_audio_batch(audio_items) if audios: - vllm_mm_data["audio"] = _normalize_optional_media_batch( - audios, - "audio", - ) + vllm_mm_data["audio"] = audios[0] if len(audios) == 1 else audios if ( video_items and mm_processor_kwargs and mm_processor_kwargs.get("use_audio_in_video", False) ): - if any( - isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item - for item in audio_items - ): - raise ValueError( - "use_audio_in_video cannot be combined with UUID-only " - "standalone audio because vLLM cannot represent cached " - "and media-backed audio slots in one batch." - ) video_audios = [] for item in video_items: url = item.get(URL_VARIANT_KEY) if isinstance(item, dict) else None if not url: raise ValueError( "use_audio_in_video requires all video items to be " - "URL-based. UUID-only and frontend-decoded video " - "items cannot provide an audio stream." + "URL-based. Got a non-URL video item (e.g. frontend-" + "decoded). Audio extraction from decoded video data " + "is not yet supported." ) try: video_audios.append(await self.audio_loader.load_audio(url)) @@ -648,12 +598,6 @@ def build_tokens_prompt( request.get("multi_modal_uuids"), self.use_unified_vision_chunk, ) - if mm_uuids is not None: - _append_video_audio_uuid_slots( - mm_uuids, - request, - mm_processor_kwargs, - ) if mm_uuids is None: mm_uuids = _build_forwarded_mm_uuids( extra_args, diff --git a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py index ea77d87d3e45..fa83ee0f289b 100644 --- a/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py +++ b/components/src/dynamo/vllm/tests/multimodal_utils/test_vllm_request_processor.py @@ -264,7 +264,7 @@ async def test_audio_in_video_preserves_order_and_merges_standalone_audio(): @pytest.mark.parametrize( ("video_item", "audio_error", "message"), [ - ({"Decoded": {"shape": [2, 4, 4, 3]}}, None, "URL-based"), + ({"Decoded": {"shape": [2, 4, 4, 3]}}, None, "non-URL video item"), ( {"Url": "https://example.com/silent.mp4"}, RuntimeError("no audio stream"), @@ -287,84 +287,6 @@ async def test_audio_in_video_rejects_unusable_audio(video_item, audio_error, me ) -@pytest.mark.asyncio -async def test_audio_in_video_rejects_uuid_only_standalone_audio(): - processor = _processor() - processor.video_loader.load_video_batch.return_value = [object()] - processor.audio_loader.load_audio_batch.return_value = [None] - - with pytest.raises(ValueError, match="UUID-only standalone audio"): - await processor.extract_multimodal_data( - { - "multi_modal_data": { - "video_url": [{"Url": "https://example.com/video.mp4"}], - "audio_url": [{"UuidOnly": "cached-audio"}], - } - }, - "request-video-cached-audio", - None, - {"use_audio_in_video": True}, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("request_key", "result_key", "loader_name"), - [ - ("video_url", "video", "video_loader"), - ("audio_url", "audio", "audio_loader"), - ], -) -async def test_uuid_only_video_audio_use_modality_level_none( - request_key, - result_key, - loader_name, -): - processor = _processor() - loader = getattr(processor, loader_name) - getattr(loader, f"load_{result_key}_batch").return_value = [None] - - result = await processor.extract_multimodal_data( - {"multi_modal_data": {request_key: [{"UuidOnly": "cached-key"}]}}, - "request-uuid-only", - None, - ) - - assert result == {result_key: None} - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("request_key", "result_key", "loader_name"), - [ - ("video_url", "video", "video_loader"), - ("audio_url", "audio", "audio_loader"), - ], -) -async def test_mixed_media_and_uuid_only_video_audio_are_rejected( - request_key, - result_key, - loader_name, -): - processor = _processor() - loader = getattr(processor, loader_name) - getattr(loader, f"load_{result_key}_batch").return_value = [object(), None] - - with pytest.raises(ValueError, match=f"Mixed .* UUID-only {result_key}"): - await processor.extract_multimodal_data( - { - "multi_modal_data": { - request_key: [ - {"Url": "https://example.com/media"}, - {"UuidOnly": "cached-key"}, - ] - } - }, - "request-mixed-uuid", - None, - ) - - def test_build_tokens_prompt_forwards_hashes_kwargs_and_vision_chunk(): processor = _processor(unified_vision_chunk=True) mm_data = {"vision_chunk": {"type": "image", "image": object(), "uuid": None}} @@ -402,54 +324,30 @@ def test_build_tokens_prompt_prefers_opaque_user_uuids_without_padding(): assert prompt["multi_modal_uuids"] == {"image": ["sku-image-a", "sku-image-b"]} -def test_build_tokens_prompt_normalizes_all_user_uuid_modalities(): +@pytest.mark.parametrize( + "unsupported_uuids", + [ + {"video_url": ["video-key"]}, + {"audio_url": "audio-key"}, + ], +) +def test_build_tokens_prompt_rejects_user_audio_video_uuids( + unsupported_uuids: dict[str, object], +) -> None: processor = _processor(unified_vision_chunk=True) - prompt = processor.build_tokens_prompt( - { - "token_ids": [1, 2, 3], - "multi_modal_uuids": { - "image_url": ["image-key"], - "video_url": ["video-key"], - "audio_url": ["audio-key"], - }, - }, - {"vision_chunk": [None], "video": [None], "audio": [None]}, - None, - ) - - assert prompt["multi_modal_uuids"] == { - "vision_chunk": ["image-key"], - "video": ["video-key"], - "audio": ["audio-key"], - } - - -def test_build_tokens_prompt_aligns_video_derived_audio_uuid_slots(): - processor = _processor() - standalone_audio = object() - video_audio = object() - - prompt = processor.build_tokens_prompt( - { - "token_ids": [1, 2, 3], - "multi_modal_data": { - "video_url": [{"Url": "https://example.com/video.mp4"}], - "audio_url": [{"Url": "https://example.com/audio.wav"}], - }, - "multi_modal_uuids": { - "video_url": ["video-key"], - "audio_url": [None], + with pytest.raises(ValueError, match="supported only for images"): + processor.build_tokens_prompt( + { + "token_ids": [1, 2, 3], + "multi_modal_uuids": { + "image_url": ["image-key"], + **unsupported_uuids, + }, }, - }, - {"video": object(), "audio": [standalone_audio, video_audio]}, - {"use_audio_in_video": True}, - ) - - assert prompt["multi_modal_uuids"] == { - "video": ["video-key"], - "audio": [None, None], - } + {"vision_chunk": [None], "video": [None], "audio": [None]}, + None, + ) def test_build_tokens_prompt_preserves_grouped_forwarded_hashes(): diff --git a/docs/features/multimodal/multimodal-vllm.md b/docs/features/multimodal/multimodal-vllm.md index ec252dadd268..af1c917e9818 100644 --- a/docs/features/multimodal/multimodal-vllm.md +++ b/docs/features/multimodal/multimodal-vllm.md @@ -104,11 +104,12 @@ curl http://localhost:8000/v1/chat/completions \ ### Reuse vLLM Multimodal Processor Cache Entries -vLLM can cache processed image, video, and audio inputs under a client-provided -opaque UUID. This extension is specific to vLLM; it is not part of the OpenAI -chat completions schema and is not supported by Dynamo's other backends. -Dynamo's SGLang and TensorRT-LLM backends reject requests containing this -`uuid` field rather than silently ignoring unsupported cache semantics. +vLLM can cache processed multimodal inputs under a client-provided opaque UUID. +Dynamo currently exposes this behavior for images only. The extension is +specific to vLLM; it is not part of the OpenAI Chat Completions API and is not +supported by Dynamo's other backends. Dynamo rejects UUIDs on audio or video, +and its SGLang and TensorRT-LLM backends reject image UUIDs rather than silently +ignoring unsupported cache semantics. To enable the cache, pass a nonzero `--mm-processor-cache-gb` value to the vLLM worker. Populate an entry by adding `uuid` beside the media field: @@ -134,21 +135,18 @@ sending the same top-level UUID: } ``` -UUIDs are opaque nonempty strings. Do not nest `uuid` inside `image_url`, -`video_url`, or `audio_url`. A UUID-only request fails on a cache miss because -it contains no media payload to process. Dynamo also rejects a media content -part that has neither a URL nor a UUID. +UUIDs are opaque nonempty strings. The top-level field is canonical. Dynamo +still accepts the deprecated nested `image_url.uuid` form for compatibility, +but nested values must be UUID-formatted and cannot express UUID-only requests. +A UUID-only request fails on a cache miss because it contains no media payload +to process. Dynamo also rejects a media content part that has neither a URL nor +a UUID. When KV-aware routing is enabled, requests with client UUIDs use text-prefix routing. Dynamo cannot convert an opaque client key into its content-derived multimodal routing hash without risking different cache identities at the router and worker. -With vLLM, a video or audio modality may be entirely URL/decoded media or -entirely UUID-only. Do not mix media-backed and UUID-only entries within one -video or audio modality batch; vLLM does not represent missing media slots in -those batch parsers. Image batches support aligned mixed slots. - **Video request:** ```bash diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index 4c51558ed815..36a623d69fcd 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -1750,7 +1750,7 @@ dependencies = [ [[package]] name = "dynamo-protocols" version = "2.0.2" -source = "git+https://github.com/furionw/frontend-crates.git?rev=31cd250192dbca6b3005bec8170e6f95d2bc0abf#31cd250192dbca6b3005bec8170e6f95d2bc0abf" +source = "git+https://github.com/furionw/frontend-crates.git?rev=522a09c319772e3e8c4250f9afd917e7a460f2cc#522a09c319772e3e8c4250f9afd917e7a460f2cc" dependencies = [ "async-openai", "derive_builder", @@ -1760,6 +1760,7 @@ dependencies = [ "thiserror 2.0.18", "tracing", "url", + "uuid", ] [[package]] diff --git a/lib/bindings/kvbm/Cargo.toml b/lib/bindings/kvbm/Cargo.toml index dc05697999f5..68e0c6b0855e 100644 --- a/lib/bindings/kvbm/Cargo.toml +++ b/lib/bindings/kvbm/Cargo.toml @@ -63,4 +63,4 @@ cudarc = { version = "=0.19.3", features = ["cuda-version-from-build-system", "f [patch.crates-io] # Temporary until ai-dynamo/frontend-crates#119 is released. -dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" } +dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "522a09c319772e3e8c4250f9afd917e7a460f2cc" } diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index d40e8d6a6c0e..262efab8a5b8 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2363,7 +2363,7 @@ dependencies = [ [[package]] name = "dynamo-protocols" version = "2.0.2" -source = "git+https://github.com/furionw/frontend-crates.git?rev=31cd250192dbca6b3005bec8170e6f95d2bc0abf#31cd250192dbca6b3005bec8170e6f95d2bc0abf" +source = "git+https://github.com/furionw/frontend-crates.git?rev=522a09c319772e3e8c4250f9afd917e7a460f2cc#522a09c319772e3e8c4250f9afd917e7a460f2cc" dependencies = [ "async-openai", "derive_builder", @@ -2373,6 +2373,7 @@ dependencies = [ "thiserror 2.0.18", "tracing", "url", + "uuid", ] [[package]] diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index 0f7ff44585d2..24a186e1e18d 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -121,4 +121,4 @@ strip = false [patch.crates-io] # Temporary until ai-dynamo/frontend-crates#119 is released. -dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" } +dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "522a09c319772e3e8c4250f9afd917e7a460f2cc" } diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index f233fc642329..47cb0f4c5f02 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -1244,6 +1244,7 @@ impl OpenAIPreprocessor { } } + #[allow(deprecated)] pub async fn gather_multi_modal_data< R: OAIChatLikeRequest + MediaRequestExt + NvExtProvider, >( @@ -1311,21 +1312,50 @@ impl OpenAIPreprocessor { }; for content_part in content_parts.iter() { let (type_str, url, uuid) = match content_part { - ChatCompletionRequestUserMessageContentPart::ImageUrl(part) => ( - "image_url", - part.image_url.as_ref().and_then(|media| media.url.clone()), - part.uuid.clone(), - ), - ChatCompletionRequestUserMessageContentPart::VideoUrl(part) => ( - "video_url", - part.video_url.as_ref().and_then(|media| media.url.clone()), - part.uuid.clone(), - ), - ChatCompletionRequestUserMessageContentPart::AudioUrl(part) => ( - "audio_url", - part.audio_url.as_ref().and_then(|media| media.url.clone()), - part.uuid.clone(), - ), + ChatCompletionRequestUserMessageContentPart::ImageUrl(part) => { + let legacy_uuid = part + .image_url + .as_ref() + .and_then(|media| media.uuid.as_ref()) + .map(ToString::to_string); + ( + "image_url", + part.image_url.as_ref().map(|media| media.url.clone()), + part.uuid.clone().or(legacy_uuid), + ) + } + ChatCompletionRequestUserMessageContentPart::VideoUrl(part) => { + let has_legacy_uuid = part + .video_url + .as_ref() + .is_some_and(|media| media.uuid.is_some()); + if part.uuid.is_some() || has_legacy_uuid { + bail!( + "multimodal cache UUIDs are currently supported only for image_url parts with vLLM" + ); + } + ( + "video_url", + part.video_url.as_ref().map(|media| media.url.clone()), + None, + ) + } + ChatCompletionRequestUserMessageContentPart::AudioUrl(part) => { + let has_legacy_uuid = part + .audio_url + .as_ref() + .is_some_and(|media| media.uuid.is_some()); + if part.uuid.is_some() || has_legacy_uuid { + bail!( + "multimodal cache UUIDs are currently supported only for image_url parts with vLLM" + ); + } + ( + "audio_url", + part.audio_url.as_ref().map(|media| media.url.clone()), + None, + ) + } _ => continue, }; @@ -1341,10 +1371,12 @@ impl OpenAIPreprocessor { let modality = type_str.to_string(); let slot_idx = media_map.entry(modality.clone()).or_default().len(); any_user_uuid |= uuid.is_some(); - uuid_map - .entry(modality.clone()) - .or_default() - .push(uuid.clone()); + if type_str == "image_url" { + uuid_map + .entry(modality.clone()) + .or_default() + .push(uuid.clone()); + } match (url, uuid) { (Some(url), _) => { diff --git a/lib/llm/src/preprocessor/media/loader.rs b/lib/llm/src/preprocessor/media/loader.rs index 5c57a54e9b9e..7696fdaf789a 100644 --- a/lib/llm/src/preprocessor/media/loader.rs +++ b/lib/llm/src/preprocessor/media/loader.rs @@ -473,10 +473,7 @@ impl MediaLoader { // cache to stay correct; in practice it's None on the common // path so the hit rate is unaffected. if media_io_kwargs.is_none() - && let Some(url) = image_part - .image_url - .as_ref() - .and_then(|media| media.url.as_ref()) + && let Some(url) = image_part.image_url.as_ref().map(|media| &media.url) { let key = Self::cache_key(url.as_str()); if let Some(hit) = cache.lock().get(&key) { @@ -498,7 +495,7 @@ impl MediaLoader { let url = image_part .image_url .as_ref() - .and_then(|media| media.url.as_ref()) + .map(|media| &media.url) .ok_or_else(|| { anyhow::anyhow!( "Cannot decode an image content part without a URL; UUID-only parts must be resolved by the backend cache" @@ -529,11 +526,9 @@ impl MediaLoader { let url = video_part .video_url .as_ref() - .and_then(|media| media.url.as_ref()) + .map(|media| &media.url) .ok_or_else(|| { - anyhow::anyhow!( - "Cannot decode a video content part without a URL; UUID-only parts must be resolved by the backend cache" - ) + anyhow::anyhow!("Cannot decode a video content part without a URL") })?; self.media_fetcher .check_if_url_allowed_with_dns(url) @@ -563,10 +558,7 @@ impl MediaLoader { if let (Some(cache), ChatCompletionRequestUserMessageContentPart::ImageUrl(image_part)) = (self.cache.as_ref(), oai_content_part) && media_io_kwargs.is_none() - && let Some(url) = image_part - .image_url - .as_ref() - .and_then(|media| media.url.as_ref()) + && let Some(url) = image_part.image_url.as_ref().map(|media| &media.url) { let key = Self::cache_key(url.as_str()); let bytes = descriptor_bytes(&rdma_descriptor); diff --git a/lib/llm/src/protocols/anthropic/types.rs b/lib/llm/src/protocols/anthropic/types.rs index e20a50871e3f..7419feccbc82 100644 --- a/lib/llm/src/protocols/anthropic/types.rs +++ b/lib/llm/src/protocols/anthropic/types.rs @@ -168,6 +168,7 @@ impl TryFrom for NvCreateChatCompletionRequest { /// Convert user-role content blocks into chat completion messages. /// Tool results become separate Tool messages; text/image blocks become user messages. +#[allow(deprecated)] fn convert_user_blocks( blocks: &[AnthropicContentBlock], messages: &mut Vec, @@ -199,8 +200,9 @@ fn convert_user_blocks( content_parts.push(ChatCompletionRequestUserMessageContentPart::ImageUrl( ChatCompletionRequestMessageContentPartImage { image_url: Some(ImageUrl { - url: Some(url), + url, detail: None, + uuid: None, }), uuid: None, }, @@ -1913,7 +1915,12 @@ mod tests { // Second part: image with data URI match &parts[1] { ChatCompletionRequestUserMessageContentPart::ImageUrl(img) => { - let url_str = img.image_url.url.to_string(); + let url_str = img + .image_url + .as_ref() + .expect("converted image must contain a URL") + .url + .to_string(); assert!( url_str.starts_with("data:image/png;base64,"), "expected data URI, got: {url_str}" diff --git a/lib/llm/src/protocols/common/preprocessor.rs b/lib/llm/src/protocols/common/preprocessor.rs index df5fee1a4d3e..ab05d6c56a84 100644 --- a/lib/llm/src/protocols/common/preprocessor.rs +++ b/lib/llm/src/protocols/common/preprocessor.rs @@ -152,14 +152,14 @@ pub enum MultimodalData { #[serde(rename(serialize = "Url"))] RawUrl(String), Decoded(RdmaMediaDataDescriptor), - /// Payload-free slot resolved by the backend's multimodal processor cache. + /// Payload-free image slot resolved by vLLM's multimodal processor cache. UuidOnly(String), } // multimodal map containing {mm_part_type: [data...]} pub type MultimodalDataMap = std::collections::HashMap>; -/// Cache UUIDs aligned positionally with each modality's data slots. +/// Image cache UUIDs aligned positionally with `image_url` data slots. pub type MultimodalUuidMap = std::collections::HashMap>>; /// [`PreprocessedRequest`] is the internal representation of an LLM request. The [`dynamo.llm-preprocessor`] @@ -189,7 +189,7 @@ pub struct PreprocessedRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_modal_data: Option, - /// User-provided multimodal cache identities aligned with `multi_modal_data`. + /// User-provided vLLM image cache identities aligned with `multi_modal_data`. #[builder(default)] #[serde(default, skip_serializing_if = "Option::is_none")] pub multi_modal_uuids: Option, diff --git a/lib/llm/src/protocols/openai/responses/mod.rs b/lib/llm/src/protocols/openai/responses/mod.rs index f4f8a5c4652d..13ee3caa445d 100644 --- a/lib/llm/src/protocols/openai/responses/mod.rs +++ b/lib/llm/src/protocols/openai/responses/mod.rs @@ -253,6 +253,7 @@ fn convert_image_detail_str(detail: &impl serde::Serialize) -> ChatImageDetail { } /// Convert a slice of InputContent to ChatCompletionRequestUserMessageContent. +#[allow(deprecated)] fn convert_input_content_to_user_content( content: &[InputContent], ) -> Result { @@ -290,8 +291,9 @@ fn convert_input_content_to_user_content( chat_parts.push(ChatCompletionRequestUserMessageContentPart::ImageUrl( ChatCompletionRequestMessageContentPartImage { image_url: Some(ImageUrl { - url: Some(url), + url, detail: Some(convert_image_detail_str(&img.detail)), + uuid: None, }), uuid: None, }, diff --git a/lib/llm/tests/preprocessor.rs b/lib/llm/tests/preprocessor.rs index ef648f55044b..cea9b5137aad 100644 --- a/lib/llm/tests/preprocessor.rs +++ b/lib/llm/tests/preprocessor.rs @@ -687,6 +687,70 @@ mod cached_multimodal_uuid { ); } + #[tokio::test] + async fn accepts_deprecated_nested_image_uuid() { + let messages = r#"[ + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": { + "url": "https://example.com/first.png", + "uuid": "92b888ad-e64a-478f-b688-5091e16544e3" + } + } + ] + } + ]"#; + let request = Request::from(messages, None, None, "test-model".to_string()); + + let (preprocessed, _, _) = make_preprocessor() + .preprocess_request(&request, None) + .await + .unwrap(); + + assert_eq!( + preprocessed.multi_modal_uuids.as_ref().unwrap()["image_url"], + vec![Some("92b888ad-e64a-478f-b688-5091e16544e3".to_string())] + ); + } + + #[tokio::test] + async fn rejects_audio_and_video_cache_uuids() { + for messages in [ + r#"[{ + "role": "user", + "content": [{ + "type": "video_url", + "video_url": {"url": "https://example.com/video.mp4"}, + "uuid": "cached-video" + }] + }]"#, + r#"[{ + "role": "user", + "content": [{ + "type": "audio_url", + "audio_url": { + "url": "https://example.com/audio.wav", + "uuid": "92b888ad-e64a-478f-b688-5091e16544e3" + } + }] + }]"#, + ] { + let request = Request::from(messages, None, None, "test-model".to_string()); + let error = make_preprocessor() + .preprocess_request(&request, None) + .await + .expect_err("audio and video cache UUIDs must be rejected"); + + assert!( + format!("{error:#}").contains("supported only for image_url parts with vLLM"), + "unexpected error: {error:#}" + ); + } + } + #[tokio::test] async fn rejects_media_part_without_url_or_uuid() { let messages = r#"[ From 3f9e9a1433b472911fdbfb6a0682efe3c4cdf119 Mon Sep 17 00:00:00 2001 From: "Zhuangcheng(Jesse) Gu" Date: Fri, 17 Jul 2026 22:50:03 +0000 Subject: [PATCH 4/4] chore(frontend): use published UUID passthrough crates Signed-off-by: Zhuangcheng(Jesse) Gu --- Cargo.lock | 57 +++++++++++++++++----------------- Cargo.toml | 12 +++---- lib/bindings/kvbm/Cargo.lock | 17 +++++----- lib/bindings/kvbm/Cargo.toml | 4 --- lib/bindings/python/Cargo.lock | 17 +++++----- lib/bindings/python/Cargo.toml | 6 +--- lib/llm/src/model_card.rs | 15 +++++---- 7 files changed, 61 insertions(+), 67 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ee6254da438..ce30ee523073 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,7 +121,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -132,7 +132,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1563,7 +1563,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -2302,7 +2302,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2722,9 +2722,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18f8f6a2d325675a65beb0718940124a011e5ea25ee827df19028b965296a6c" +checksum = "bf45340e7f5beb8f3055c1fb933394f05256b6548a1b1434ae3df27bfc8d2f6e" dependencies = [ "aho-corasick", "anyhow", @@ -2760,8 +2760,9 @@ dependencies = [ [[package]] name = "dynamo-protocols" -version = "2.0.2" -source = "git+https://github.com/ai-dynamo/frontend-crates.git?rev=5cf0f586240cc1d1e4944b339544567a353d26b7#5cf0f586240cc1d1e4944b339544567a353d26b7" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf9875c31601458f29b83e5ed0bf63479dc3a086e4131bbee0863101f4fd5f3" dependencies = [ "async-openai", "derive_builder", @@ -2776,9 +2777,9 @@ dependencies = [ [[package]] name = "dynamo-renderer" -version = "1.3.8" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3472f4ee978f581bac985c4f5ebad55a9b1b4ccb025ed3e9df708efa56e83bfa" +checksum = "1afd1dc880ea81af098f1d0586af1ab37865c82f87d14265b7e72bf00696ffef" dependencies = [ "anyhow", "chrono", @@ -2905,9 +2906,9 @@ dependencies = [ [[package]] name = "dynamo-tokenizers" -version = "1.5.0" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb9c031ea19d99d1faf8ff304fe666eb2dbd4666fcb416af3f08ae88f5be80" +checksum = "2843cd9cd50002ae23f3df525e1c551775b7bc7c9041742539faf63bc35ea225" dependencies = [ "aho-corasick", "anyhow", @@ -3143,7 +3144,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4113,7 +4114,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.4", "system-configuration 0.6.1", "tokio", "tower-service", @@ -4449,7 +4450,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5826,7 +5827,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6989,8 +6990,8 @@ version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -7009,8 +7010,8 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ - "heck 0.4.1", - "itertools 0.10.5", + "heck 0.5.0", + "itertools 0.14.0", "log", "multimap", "petgraph 0.8.3", @@ -7031,7 +7032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -7044,7 +7045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -7234,7 +7235,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.2", "rustls 0.23.40", - "socket2 0.5.10", + "socket2 0.6.4", "thiserror 2.0.18", "tokio", "tracing", @@ -7271,7 +7272,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.4", "tracing", "windows-sys 0.60.2", ] @@ -7918,7 +7919,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8695,7 +8696,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -8961,7 +8962,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10785,7 +10786,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2c04ce93004b..c7a07701f7fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,8 +46,8 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed"] dynamo-runtime = { path = "lib/runtime", version = "1.3.0" } dynamo-llm = { path = "lib/llm", version = "1.3.0" } dynamo-rl = { path = "lib/rl", version = "1.3.0" } -dynamo-tokenizers = { version = "=1.5.0" } -dynamo-renderer = { version = "=1.3.8" } +dynamo-tokenizers = { version = "=1.5.3" } +dynamo-renderer = { version = "=2.0.0" } dynamo-tokens = { path = "lib/tokens", version = "1.3.0" } dynamo-truthy = { path = "lib/truthy", version = "1.3.0" } dynamo-data-gen = { path = "lib/data-gen", version = "1.3.0" } @@ -55,8 +55,8 @@ dynamo-kv-hashing = { path = "lib/kv-hashing", version = "1.3.0" } dynamo-memory = { path = "lib/memory", version = "1.3.0" } dynamo-mocker = { path = "lib/mocker", version = "1.3.0" } dynamo-kv-router = { path = "lib/kv-router", version = "1.3.0" } -dynamo-protocols = { git = "https://github.com/ai-dynamo/frontend-crates.git", rev = "5cf0f586240cc1d1e4944b339544567a353d26b7" } -dynamo-parsers = { version = "5.0.0" } +dynamo-protocols = { version = "3.0.0" } +dynamo-parsers = { version = "5.1.0" } # Published on crates.io. To see all available versions: # web: https://crates.io/crates/dynamo-parsers-v2/versions # sparse index: curl https://index.crates.io/dy/na/dynamo-parsers-v2 (one JSON line per release; read the "vers" field) @@ -182,10 +182,6 @@ tokio-rustls = { version = "0.26" } rustls = { version = "0.23", default-features = false, features = ["ring", "std"] } rustls-pemfile = { version = "2" } -[patch.crates-io] -# Temporary until UUID passthrough support is released in dynamo-protocols. -dynamo-protocols = { git = "https://github.com/ai-dynamo/frontend-crates.git", rev = "5cf0f586240cc1d1e4944b339544567a353d26b7" } - [profile.dev.package] insta.opt-level = 3 diff --git a/lib/bindings/kvbm/Cargo.lock b/lib/bindings/kvbm/Cargo.lock index dbcade2fe0ca..4d5ea47221aa 100644 --- a/lib/bindings/kvbm/Cargo.lock +++ b/lib/bindings/kvbm/Cargo.lock @@ -1713,9 +1713,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18f8f6a2d325675a65beb0718940124a011e5ea25ee827df19028b965296a6c" +checksum = "bf45340e7f5beb8f3055c1fb933394f05256b6548a1b1434ae3df27bfc8d2f6e" dependencies = [ "aho-corasick", "anyhow", @@ -1751,8 +1751,9 @@ dependencies = [ [[package]] name = "dynamo-protocols" -version = "2.0.2" -source = "git+https://github.com/ai-dynamo/frontend-crates.git?rev=5cf0f586240cc1d1e4944b339544567a353d26b7#5cf0f586240cc1d1e4944b339544567a353d26b7" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf9875c31601458f29b83e5ed0bf63479dc3a086e4131bbee0863101f4fd5f3" dependencies = [ "async-openai", "derive_builder", @@ -1767,9 +1768,9 @@ dependencies = [ [[package]] name = "dynamo-renderer" -version = "1.3.8" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3472f4ee978f581bac985c4f5ebad55a9b1b4ccb025ed3e9df708efa56e83bfa" +checksum = "1afd1dc880ea81af098f1d0586af1ab37865c82f87d14265b7e72bf00696ffef" dependencies = [ "anyhow", "chrono", @@ -1866,9 +1867,9 @@ dependencies = [ [[package]] name = "dynamo-tokenizers" -version = "1.5.0" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb9c031ea19d99d1faf8ff304fe666eb2dbd4666fcb416af3f08ae88f5be80" +checksum = "2843cd9cd50002ae23f3df525e1c551775b7bc7c9041742539faf63bc35ea225" dependencies = [ "aho-corasick", "anyhow", diff --git a/lib/bindings/kvbm/Cargo.toml b/lib/bindings/kvbm/Cargo.toml index 70568f00b0c6..6c713f6bbe6b 100644 --- a/lib/bindings/kvbm/Cargo.toml +++ b/lib/bindings/kvbm/Cargo.toml @@ -60,7 +60,3 @@ dlpark = { version = "0.5", features = ["pyo3", "half"], optional = true } cudarc = { version = "=0.19.8", features = ["cuda-version-from-build-system", "fallback-latest"], optional = true } [dev-dependencies] - -[patch.crates-io] -# Temporary until UUID passthrough support is released in dynamo-protocols. -dynamo-protocols = { git = "https://github.com/ai-dynamo/frontend-crates.git", rev = "5cf0f586240cc1d1e4944b339544567a353d26b7" } diff --git a/lib/bindings/python/Cargo.lock b/lib/bindings/python/Cargo.lock index 09da9a551ed6..87789a0a097e 100644 --- a/lib/bindings/python/Cargo.lock +++ b/lib/bindings/python/Cargo.lock @@ -2326,9 +2326,9 @@ dependencies = [ [[package]] name = "dynamo-parsers" -version = "5.0.0" +version = "5.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c18f8f6a2d325675a65beb0718940124a011e5ea25ee827df19028b965296a6c" +checksum = "bf45340e7f5beb8f3055c1fb933394f05256b6548a1b1434ae3df27bfc8d2f6e" dependencies = [ "aho-corasick", "anyhow", @@ -2364,8 +2364,9 @@ dependencies = [ [[package]] name = "dynamo-protocols" -version = "2.0.2" -source = "git+https://github.com/ai-dynamo/frontend-crates.git?rev=5cf0f586240cc1d1e4944b339544567a353d26b7#5cf0f586240cc1d1e4944b339544567a353d26b7" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aaf9875c31601458f29b83e5ed0bf63479dc3a086e4131bbee0863101f4fd5f3" dependencies = [ "async-openai", "derive_builder", @@ -2420,9 +2421,9 @@ dependencies = [ [[package]] name = "dynamo-renderer" -version = "1.3.8" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3472f4ee978f581bac985c4f5ebad55a9b1b4ccb025ed3e9df708efa56e83bfa" +checksum = "1afd1dc880ea81af098f1d0586af1ab37865c82f87d14265b7e72bf00696ffef" dependencies = [ "anyhow", "chrono", @@ -2542,9 +2543,9 @@ dependencies = [ [[package]] name = "dynamo-tokenizers" -version = "1.5.0" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fb9c031ea19d99d1faf8ff304fe666eb2dbd4666fcb416af3f08ae88f5be80" +checksum = "2843cd9cd50002ae23f3df525e1c551775b7bc7c9041742539faf63bc35ea225" dependencies = [ "aho-corasick", "anyhow", diff --git a/lib/bindings/python/Cargo.toml b/lib/bindings/python/Cargo.toml index b9c33e50f8eb..e33281b43017 100644 --- a/lib/bindings/python/Cargo.toml +++ b/lib/bindings/python/Cargo.toml @@ -49,7 +49,7 @@ mm-routing = ["dynamo-llm/mm-routing"] aiconfigurator-core = { git = "https://github.com/ai-dynamo/aiconfigurator.git", rev = "ffbab15797d3a3f14c382e937c0ab0c1c5cbc530", package = "aiconfigurator-core", optional = true } dynamo-runtime = { path = "../../runtime" } dynamo-mocker = { path = "../../mocker" } -dynamo-parsers = { version = "5.0.0" } +dynamo-parsers = { version = "5.1.0" } dynamo-backend-common = { path = "../../backend-common" } dynamo-sglang-sidecar = { path = "../../backend/sglang-sidecar" } @@ -118,7 +118,3 @@ tracing-subscriber = { version = "0.3", features = ["registry"] } inherits = "release" debug = true strip = false - -[patch.crates-io] -# Temporary until UUID passthrough support is released in dynamo-protocols. -dynamo-protocols = { git = "https://github.com/ai-dynamo/frontend-crates.git", rev = "5cf0f586240cc1d1e4944b339544567a353d26b7" } diff --git a/lib/llm/src/model_card.rs b/lib/llm/src/model_card.rs index eb62b176ccef..d20c0e61a9c1 100644 --- a/lib/llm/src/model_card.rs +++ b/lib/llm/src/model_card.rs @@ -61,9 +61,12 @@ fn instrumented_tokenizer_cache( cache_bytes: usize, cache_extend: bool, model: &str, -) -> Arc { - Arc::new( - crate::tokenizers::CachedTokenizer::new(raw, special_tokens, cache_bytes) +) -> Result> { + let cached = crate::tokenizers::CachedTokenizer::new(raw, special_tokens, cache_bytes) + .context("failed to initialize tokenizer prefix cache")?; + + Ok(Arc::new( + cached .with_extend(cache_extend) .with_observer( Arc::new(|| { @@ -74,7 +77,7 @@ fn instrumented_tokenizer_cache( }), ) .with_token_observer(tokenizer_cache_token_observer(model)), - ) + )) } /// Identify model deployment cards in the key-value store @@ -1240,7 +1243,7 @@ impl ModelDeploymentCard { cache_bytes, cache_extend, self.name(), - ) + )? } else { raw } @@ -1272,7 +1275,7 @@ impl ModelDeploymentCard { cache_bytes, cache_extend, self.name(), - ) + )? } else { raw }