Skip to content
Closed
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ dynamo-kv-hashing = { path = "lib/kv-hashing", version = "1.3.0" }
dynamo-memory = { path = "lib/memory", version = "1.3.0" }
dynamo-mocker = { path = "lib/mocker", version = "1.3.0" }
dynamo-kv-router = { path = "lib/kv-router", version = "1.3.0" }
dynamo-protocols = { version = "=2.0.2" }
dynamo-protocols = { git = "https://github.com/ai-dynamo/frontend-crates.git", rev = "5cf0f586240cc1d1e4944b339544567a353d26b7" }
dynamo-parsers = { version = "5.0.0" }
# Published on crates.io. To see all available versions:
# web: https://crates.io/crates/dynamo-parsers-v2/versions
Expand Down Expand Up @@ -182,6 +182,10 @@ tokio-rustls = { version = "0.26" }
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
rustls-pemfile = { version = "2" }

[patch.crates-io]
# Temporary until UUID passthrough support is released in dynamo-protocols.
dynamo-protocols = { git = "https://github.com/ai-dynamo/frontend-crates.git", rev = "5cf0f586240cc1d1e4944b339544567a353d26b7" }

[profile.dev.package]
insta.opt-level = 3

Expand Down
18 changes: 18 additions & 0 deletions components/src/dynamo/common/multimodal/cache_uuid.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Backend capability guard for client-provided multimodal cache UUIDs."""


def reject_unsupported_multimodal_uuids(
multi_modal_uuids: object,
*,
backend: str,
) -> None:
if not multi_modal_uuids:
return

raise ValueError(
"Image UUID caching is currently supported only by the vLLM "
f"backend; {backend} does not support the `uuid` field"
)
35 changes: 29 additions & 6 deletions components/src/dynamo/common/multimodal/image_loader.py
Comment thread
Chokoyo marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import os
from collections import OrderedDict
from io import BytesIO
from typing import Any, Dict, Final, List
from typing import Any, Coroutine, Dict, Final, List
from urllib.parse import urlparse

from PIL import Image
Expand All @@ -28,6 +28,7 @@
# Constants for multimodal data variants
URL_VARIANT_KEY: Final = "Url"
DECODED_VARIANT_KEY: Final = "Decoded"
UUID_ONLY_VARIANT_KEY: Final = "UuidOnly"


def _create_nixl_connector() -> Any:
Expand Down Expand Up @@ -267,35 +268,57 @@ async def load_image_batch(
Exception: If any image fails to load for any other reason
ValueError: If enable_frontend_decoding=True but nixl_connector is None
"""
image_futures = []
image_futures: list[Coroutine[Any, Any, Image.Image]] = []
slot_to_future_idx: list[int | None] = []

for item in image_mm_items:
for idx, item in enumerate(image_mm_items):
if isinstance(item, dict) and URL_VARIANT_KEY in item:
# URL path: download and decode in Python backend
url = item[URL_VARIANT_KEY]
slot_to_future_idx.append(len(image_futures))
image_futures.append(self.load_image(url))
logger.debug(f"Preparing to load image from URL: {url[:80]}...")
elif isinstance(item, dict) and DECODED_VARIANT_KEY in item:
if self._enable_frontend_decoding:
metadata = item[DECODED_VARIANT_KEY]
if self._nixl_connector is None:
raise RuntimeError("NIXL connector is not initialized")
slot_to_future_idx.append(len(image_futures))
image_futures.append(self._read_and_convert_nixl_image(metadata))
else:
logger.error(
"Received Decoded multimodal data but enable_frontend_decoding=False. "
"Set enable_frontend_decoding=True to enable NIXL RDMA image transfer."
)
raise ValueError("Could not load decoded media from frontend")
elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item:
slot_to_future_idx.append(None)
else:
raise ValueError(
f"Invalid image multimodal item at index {idx}. "
"Expected dict with 'Url', 'Decoded', or 'UuidOnly' key."
)

# Process images in parallel
results = await asyncio.gather(*image_futures, return_exceptions=True)
loaded_images = []
loaded_images: list[Image.Image | None] = []
collective_exceptions = ""
status_error: HttpStatusError | None = None
url_error: UrlValidationError | None = None
for media_item, result in zip(image_mm_items, results):
if isinstance(result, Exception):
for media_item, future_idx in zip(
image_mm_items, slot_to_future_idx, strict=True
):
if future_idx is None:
loaded_images.append(None)
continue
result = results[future_idx]
if isinstance(result, BaseException):
# asyncio.gather(..., return_exceptions=True) may return a
# cancellation (a BaseException, but not an Exception). Keep
# cancellation semantics instead of folding it into a batch
# image-loading error.
if not isinstance(result, Exception):
raise result
source = media_item.get(URL_VARIANT_KEY, "decoded")
logger.error(f"Failed to load image from {source[:80]}...: {result}")
collective_exceptions += (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,37 @@ async def test_unsupported_format_batch_url_raises_415(loader: ImageLoader) -> N
assert exc_info.value.status == 415


async def test_uuid_only_image_slots_preserve_batch_alignment(
loader: ImageLoader,
) -> None:
first = Image.new("RGB", (1, 1), color="red")
second = Image.new("RGB", (1, 1), color="blue")
loader.load_image = AsyncMock( # type: ignore[method-assign]
side_effect=[first, second]
)

images = await loader.load_image_batch(
[
{"Url": "https://example.com/first.png"},
{"UuidOnly": "cached-image"},
{"Url": "https://example.com/second.png"},
]
)

assert images == [first, None, second]


async def test_batch_propagates_cancellation(loader: ImageLoader) -> None:
loader.load_image = AsyncMock( # type: ignore[method-assign]
side_effect=asyncio.CancelledError
)

with pytest.raises(asyncio.CancelledError):
await loader.load_image_batch(
[{URL_VARIANT_KEY: "https://example.com/image.png"}]
)


async def test_unsupported_format_batch_data_url_raises_415(
loader: ImageLoader,
) -> None:
Expand Down
10 changes: 8 additions & 2 deletions components/src/dynamo/frontend/sglang_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
from sglang.srt.utils.hf_transformers_utils import get_tokenizer

from dynamo._internal import ModelDeploymentCard
from dynamo.common.multimodal.cache_uuid import reject_unsupported_multimodal_uuids
from dynamo.frontend.frontend_args import FrontendConfig
from dynamo.llm import ModelCardInstanceId, PythonAsyncEngine, RoutedEngine
from dynamo.llm.exceptions import InvalidArgument, Unknown
Expand Down Expand Up @@ -327,7 +328,11 @@ def _build_dynamo_preproc(
}

# Forward multimodal URLs so the backend handler can load the media.
mm_data = extract_mm_urls(request.get("messages", []))
mm_data, mm_uuids = extract_mm_urls(request.get("messages", []))
try:
reject_unsupported_multimodal_uuids(mm_uuids, backend="SGLang")
except ValueError as exc:
raise PreprocessError(str(exc)) from exc
if mm_data:
preproc["multi_modal_data"] = mm_data

Expand Down Expand Up @@ -579,7 +584,8 @@ async def _generate_and_stream(
cumulative_output_tokens = 0
# Rust postprocessor is bypassed on this path, so emit the multimodal
# content-part counts here too (else frontend metrics report zero media).
_mm_counts = extract_mm_urls(request.get("messages", [])) or {}
_mm_counts, _ = extract_mm_urls(request.get("messages", []))
_mm_counts = _mm_counts or {}
image_count = len(_mm_counts.get("image_url", []))
video_count = len(_mm_counts.get("video_url", []))
audio_count = len(_mm_counts.get("audio_url", []))
Expand Down
137 changes: 119 additions & 18 deletions components/src/dynamo/frontend/tests/test_multimodal_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,11 @@ def test_returns_none_for_text_only():
messages = [
{"role": "user", "content": "What is the capital of France?"},
]
assert extract_mm_urls(messages) is None
assert extract_mm_urls(messages) == (None, None)


def test_returns_none_for_empty_messages():
assert extract_mm_urls([]) is None
assert extract_mm_urls([]) == (None, None)


def test_extracts_image_urls():
Expand All @@ -37,7 +37,7 @@ def test_extracts_image_urls():
}
]
result = extract_mm_urls(messages)
assert result == {"image_url": [{"Url": "https://example.com/cat.png"}]}
assert result == ({"image_url": [{"Url": "https://example.com/cat.png"}]}, None)


def test_extracts_audio_urls():
Expand All @@ -54,7 +54,10 @@ def test_extracts_audio_urls():
}
]
result = extract_mm_urls(messages)
assert result == {"audio_url": [{"Url": "data:audio/wav;base64,UklGRg=="}]}
assert result == (
{"audio_url": [{"Url": "data:audio/wav;base64,UklGRg=="}]},
None,
)


def test_extracts_video_urls():
Expand All @@ -70,7 +73,10 @@ def test_extracts_video_urls():
}
]
result = extract_mm_urls(messages)
assert result == {"video_url": [{"Url": "https://example.com/clip.mp4"}]}
assert result == (
{"video_url": [{"Url": "https://example.com/clip.mp4"}]},
None,
)


def test_extracts_mixed_modalities():
Expand All @@ -95,11 +101,14 @@ def test_extracts_mixed_modalities():
}
]
result = extract_mm_urls(messages)
assert result == {
"image_url": [{"Url": "https://example.com/img.jpg"}],
"audio_url": [{"Url": "https://example.com/audio.wav"}],
"video_url": [{"Url": "https://example.com/video.mp4"}],
}
assert result == (
{
"image_url": [{"Url": "https://example.com/img.jpg"}],
"audio_url": [{"Url": "https://example.com/audio.wav"}],
"video_url": [{"Url": "https://example.com/video.mp4"}],
},
None,
)


def test_extracts_multiple_items_per_modality():
Expand All @@ -120,12 +129,15 @@ def test_extracts_multiple_items_per_modality():
}
]
result = extract_mm_urls(messages)
assert result == {
"image_url": [
{"Url": "https://example.com/a.png"},
{"Url": "https://example.com/b.png"},
]
}
assert result == (
{
"image_url": [
{"Url": "https://example.com/a.png"},
{"Url": "https://example.com/b.png"},
]
},
None,
)


def test_ignores_non_user_messages():
Expand All @@ -142,7 +154,7 @@ def test_ignores_non_user_messages():
},
{"role": "user", "content": "Hello"},
]
assert extract_mm_urls(messages) is None
assert extract_mm_urls(messages) == (None, None)


def test_handles_malformed_content_non_dict():
Expand All @@ -162,4 +174,93 @@ def test_handles_malformed_content_non_dict():
}
]
result = extract_mm_urls(messages)
assert result == {"image_url": [{"Url": "https://example.com/ok.png"}]}
assert result == ({"image_url": [{"Url": "https://example.com/ok.png"}]}, None)


def test_extracts_url_and_uuid_only_slots_with_alignment():
messages = [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {"url": "https://example.com/a.png"},
"uuid": "image-a",
},
{
"type": "image_url",
"image_url": None,
"uuid": "image-b",
},
{
"type": "image_url",
"image_url": {"url": "https://example.com/c.png"},
},
],
}
]

assert extract_mm_urls(messages) == (
{
"image_url": [
{"Url": "https://example.com/a.png"},
{"UuidOnly": "image-b"},
{"Url": "https://example.com/c.png"},
]
},
{"image_url": ["image-a", "image-b", None]},
)


def test_accepts_deprecated_nested_image_uuid() -> None:
messages = [
{
"role": "user",
"content": [
{
"type": "image_url",
"image_url": {
"url": "https://example.com/a.png",
"uuid": "92b888ad-e64a-478f-b688-5091e16544e3",
},
}
],
}
]

assert extract_mm_urls(messages) == (
{"image_url": [{"Url": "https://example.com/a.png"}]},
{"image_url": ["92b888ad-e64a-478f-b688-5091e16544e3"]},
)


@pytest.mark.parametrize("part_type", ["audio_url", "video_url"])
@pytest.mark.parametrize("nested", [False, True])
def test_rejects_audio_video_cache_uuids(part_type: str, nested: bool) -> None:
media = {"url": "https://example.com/media", "uuid": "cached-media"}
part = {"type": part_type, part_type: media}
if not nested:
media.pop("uuid")
part["uuid"] = "cached-media"

with pytest.raises(ValueError, match="supported only for image_url.*vLLM"):
extract_mm_urls([{"role": "user", "content": [part]}])


@pytest.mark.parametrize(
"part",
[
{"type": "image_url", "image_url": None},
{"type": "image_url", "image_url": {}},
{
"type": "image_url",
"image_url": {"url": "https://example.com/a.png"},
"uuid": "",
},
],
)
def test_rejects_media_parts_without_valid_url_or_uuid(part):
messages = [{"role": "user", "content": [part]}]

with pytest.raises(ValueError, match="URL or uuid|non-empty string"):
extract_mm_urls(messages)
Loading
Loading