Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 5 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ dynamo-kv-hashing = { path = "lib/kv-hashing", version = "1.3.0" }
dynamo-memory = { path = "lib/memory", version = "1.3.0" }
dynamo-mocker = { path = "lib/mocker", version = "1.3.0" }
dynamo-kv-router = { path = "lib/kv-router", version = "1.3.0", features = ["metrics", "runtime-protocols"] }
dynamo-protocols = { version = "=2.0.2" }
dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" }
dynamo-parsers = { version = "5.0.0" }
# Published on crates.io. To see all available versions:
# web: https://crates.io/crates/dynamo-parsers-v2/versions
Expand Down Expand Up @@ -180,6 +180,10 @@ tokio-rustls = { version = "0.26" }
rustls = { version = "0.23", default-features = false, features = ["ring", "std"] }
rustls-pemfile = { version = "2" }

[patch.crates-io]
# Temporary until ai-dynamo/frontend-crates#119 is released.
dynamo-protocols = { git = "https://github.com/furionw/frontend-crates.git", rev = "31cd250192dbca6b3005bec8170e6f95d2bc0abf" }

[profile.dev.package]
insta.opt-level = 3

Expand Down
20 changes: 16 additions & 4 deletions components/src/dynamo/common/multimodal/audio_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
# Constants for multimodal data variants
URL_VARIANT_KEY: Final = "Url"
DECODED_VARIANT_KEY: Final = "Decoded"
UUID_ONLY_VARIANT_KEY: Final = "UuidOnly"


def _create_nixl_connector() -> Any:
Expand Down Expand Up @@ -170,7 +171,7 @@ async def _load_decoded_audio(
async def load_audio_batch(
self,
audio_mm_items: List[Dict[str, Any]],
) -> List[tuple[np.ndarray, float]]:
) -> List[tuple[np.ndarray, float] | None]:
"""Load a batch of audio files from multimodal data items.

Supports two paths:
Expand All @@ -182,31 +183,42 @@ async def load_audio_batch(
List of (waveform, sample_rate) tuples.
"""
audio_futures: List[Awaitable[tuple[np.ndarray, float]]] = []
slot_to_future_idx: list[int | None] = []

for idx, item in enumerate(audio_mm_items):
if isinstance(item, dict) and URL_VARIANT_KEY in item:
url = item[URL_VARIANT_KEY]
slot_to_future_idx.append(len(audio_futures))
audio_futures.append(self.load_audio(url))
logger.debug("Preparing to load audio from URL: %s...", url[:80])
elif isinstance(item, dict) and DECODED_VARIANT_KEY in item:
if self._enable_frontend_decoding:
metadata = item[DECODED_VARIANT_KEY]
slot_to_future_idx.append(len(audio_futures))
audio_futures.append(self._load_decoded_audio(metadata))
else:
raise ValueError(
"Received decoded audio data but enable_frontend_decoding=False. "
"Enable frontend decoding to transfer decoded audio via NIXL."
)
elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item:
slot_to_future_idx.append(None)
else:
raise ValueError(
f"Invalid audio multimodal item at index {idx}. "
"Expected dict with 'Url' or 'Decoded' key."
"Expected dict with 'Url', 'Decoded', or 'UuidOnly' key."
)

results = await asyncio.gather(*audio_futures, return_exceptions=True)
loaded_audio: list[tuple[np.ndarray, float]] = []
loaded_audio: list[tuple[np.ndarray, float] | None] = []
collective_exceptions: list[str] = []
for media_item, result in zip(audio_mm_items, results, strict=True):
for media_item, future_idx in zip(
audio_mm_items, slot_to_future_idx, strict=True
):
if future_idx is None:
loaded_audio.append(None)
continue
result = results[future_idx]
if isinstance(result, BaseException):
if isinstance(result, asyncio.CancelledError):
raise result
Expand Down
21 changes: 19 additions & 2 deletions components/src/dynamo/common/multimodal/image_loader.py
Comment thread
Chokoyo marked this conversation as resolved.
Resolved
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
# Constants for multimodal data variants
URL_VARIANT_KEY: Final = "Url"
DECODED_VARIANT_KEY: Final = "Decoded"
UUID_ONLY_VARIANT_KEY: Final = "UuidOnly"


def _create_nixl_connector() -> Any:
Expand Down Expand Up @@ -268,33 +269,49 @@ async def load_image_batch(
ValueError: If enable_frontend_decoding=True but nixl_connector is None
"""
image_futures = []
slot_to_future_idx: list[int | None] = []

for item in image_mm_items:
for idx, item in enumerate(image_mm_items):
if isinstance(item, dict) and URL_VARIANT_KEY in item:
# URL path: download and decode in Python backend
url = item[URL_VARIANT_KEY]
slot_to_future_idx.append(len(image_futures))
image_futures.append(self.load_image(url))
logger.debug(f"Preparing to load image from URL: {url[:80]}...")
elif isinstance(item, dict) and DECODED_VARIANT_KEY in item:
if self._enable_frontend_decoding:
metadata = item[DECODED_VARIANT_KEY]
if self._nixl_connector is None:
raise RuntimeError("NIXL connector is not initialized")
slot_to_future_idx.append(len(image_futures))
image_futures.append(self._read_and_convert_nixl_image(metadata))
else:
logger.error(
"Received Decoded multimodal data but enable_frontend_decoding=False. "
"Set enable_frontend_decoding=True to enable NIXL RDMA image transfer."
)
raise ValueError("Could not load decoded media from frontend")
elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item:
slot_to_future_idx.append(None)
else:
raise ValueError(
f"Invalid image multimodal item at index {idx}. "
"Expected dict with 'Url', 'Decoded', or 'UuidOnly' key."
)

# Process images in parallel
results = await asyncio.gather(*image_futures, return_exceptions=True)
loaded_images = []
collective_exceptions = ""
status_error: HttpStatusError | None = None
url_error: UrlValidationError | None = None
for media_item, result in zip(image_mm_items, results):
for media_item, future_idx in zip(
image_mm_items, slot_to_future_idx, strict=True
):
if future_idx is None:
loaded_images.append(None)
continue
result = results[future_idx]
if isinstance(result, Exception):
source = media_item.get(URL_VARIANT_KEY, "decoded")
logger.error(f"Failed to load image from {source[:80]}...: {result}")
Expand Down
25 changes: 21 additions & 4 deletions components/src/dynamo/common/multimodal/video_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@

URL_VARIANT_KEY: Final = "Url"
DECODED_VARIANT_KEY: Final = "Decoded"
UUID_ONLY_VARIANT_KEY: Final = "UuidOnly"


def _create_nixl_connector() -> Any:
Expand Down Expand Up @@ -161,28 +162,44 @@ async def _load_decoded_video(
async def load_video_batch(
self,
video_mm_items: List[Dict[str, Any]],
) -> List[tuple[np.ndarray, Dict[str, Any]]]:
) -> List[tuple[np.ndarray, Dict[str, Any]] | None]:
video_futures: List[Awaitable[tuple[np.ndarray, Dict[str, Any]]]] = []
slot_to_future_idx: list[int | None] = []

for item in video_mm_items:
for idx, item in enumerate(video_mm_items):
if isinstance(item, dict) and URL_VARIANT_KEY in item:
url = item[URL_VARIANT_KEY]
slot_to_future_idx.append(len(video_futures))
video_futures.append(self.load_video(url))
logger.debug("Preparing to load video from URL: %s...", url[:80])
elif isinstance(item, dict) and DECODED_VARIANT_KEY in item:
if self._enable_frontend_decoding:
metadata = item[DECODED_VARIANT_KEY]
slot_to_future_idx.append(len(video_futures))
video_futures.append(self._load_decoded_video(metadata))
else:
raise ValueError(
"Received decoded video data but enable_frontend_decoding=False. "
"Enable frontend decoding to transfer decoded video frames via NIXL."
)
elif isinstance(item, dict) and UUID_ONLY_VARIANT_KEY in item:
slot_to_future_idx.append(None)
else:
raise ValueError(
f"Invalid video multimodal item at index {idx}. "
"Expected dict with 'Url', 'Decoded', or 'UuidOnly' key."
)

results = await asyncio.gather(*video_futures, return_exceptions=True)
loaded_videos: list[tuple[np.ndarray, Dict[str, Any]]] = []
loaded_videos: list[tuple[np.ndarray, Dict[str, Any]] | None] = []
collective_exceptions: list[str] = []
for media_item, result in zip(video_mm_items, results):
for media_item, future_idx in zip(
video_mm_items, slot_to_future_idx, strict=True
):
if future_idx is None:
loaded_videos.append(None)
continue
result = results[future_idx]
if isinstance(result, BaseException):
if isinstance(result, asyncio.CancelledError):
raise result
Expand Down
24 changes: 24 additions & 0 deletions components/src/dynamo/common/tests/multimodal/test_audio_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,3 +136,27 @@ async def test_load_audio_batch_reads_decoded_variant(monkeypatch):
decoded_item,
return_metadata=True,
)


@pytest.mark.asyncio
async def test_uuid_only_audio_slots_preserve_batch_alignment():
loader = AudioLoader()
first = (np.zeros(8, dtype=np.float32), 16000.0)
second = (np.ones(8, dtype=np.float32), 44100.0)
loader.load_audio = AsyncMock( # type: ignore[method-assign]
side_effect=[first, second]
)

audios = await loader.load_audio_batch(
[
{"Url": "https://example.com/first.wav"},
{"UuidOnly": "cached-audio"},
{"Url": "https://example.com/second.wav"},
]
)

assert audios[0] is not None
assert audios[1] is None
assert audios[2] is not None
np.testing.assert_array_equal(audios[0][0], first[0])
np.testing.assert_array_equal(audios[2][0], second[0])
Original file line number Diff line number Diff line change
Expand Up @@ -260,6 +260,26 @@ async def test_unsupported_format_batch_url_raises_415(loader: ImageLoader) -> N
assert exc_info.value.status == 415


async def test_uuid_only_image_slots_preserve_batch_alignment(
loader: ImageLoader,
) -> None:
first = Image.new("RGB", (1, 1), color="red")
second = Image.new("RGB", (1, 1), color="blue")
loader.load_image = AsyncMock( # type: ignore[method-assign]
side_effect=[first, second]
)

images = await loader.load_image_batch(
[
{"Url": "https://example.com/first.png"},
{"UuidOnly": "cached-image"},
{"Url": "https://example.com/second.png"},
]
)

assert images == [first, None, second]


async def test_unsupported_format_batch_data_url_raises_415(
loader: ImageLoader,
) -> None:
Expand Down
30 changes: 30 additions & 0 deletions components/src/dynamo/common/tests/multimodal/test_video_loader.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,3 +109,33 @@ async def test_load_video_batch_reads_decoded_variant_with_metadata(monkeypatch)
decoded_item,
return_metadata=True,
)


@pytest.mark.asyncio
async def test_uuid_only_video_slots_preserve_batch_alignment():
loader = VideoLoader()
first = (
np.zeros((1, 2, 2, 3), dtype=np.uint8),
{"fps": 2.0, "frames_indices": [0], "total_num_frames": 1},
)
second = (
np.ones((1, 2, 2, 3), dtype=np.uint8),
{"fps": 2.0, "frames_indices": [0], "total_num_frames": 1},
)
loader.load_video = AsyncMock( # type: ignore[method-assign]
side_effect=[first, second]
)

videos = await loader.load_video_batch(
[
{"Url": "https://example.com/first.mp4"},
{"UuidOnly": "cached-video"},
{"Url": "https://example.com/second.mp4"},
]
)

assert videos[0] is not None
assert videos[1] is None
assert videos[2] is not None
np.testing.assert_array_equal(videos[0][0], first[0])
np.testing.assert_array_equal(videos[2][0], second[0])
7 changes: 5 additions & 2 deletions components/src/dynamo/frontend/sglang_processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,11 @@ def _build_dynamo_preproc(
}

# Forward multimodal URLs so the backend handler can load the media.
mm_data = extract_mm_urls(request.get("messages", []))
mm_data, mm_uuids = extract_mm_urls(request.get("messages", []))
if mm_data:
preproc["multi_modal_data"] = mm_data
if mm_uuids:
preproc["multi_modal_uuids"] = mm_uuids

nvext = request.get("nvext") or {}
nvext_passthrough = {
Expand Down Expand Up @@ -575,7 +577,8 @@ async def _generate_and_stream(
cumulative_output_tokens = 0
# Rust postprocessor is bypassed on this path, so emit the multimodal
# content-part counts here too (else frontend metrics report zero media).
_mm_counts = extract_mm_urls(request.get("messages", [])) or {}
_mm_counts, _ = extract_mm_urls(request.get("messages", []))
_mm_counts = _mm_counts or {}
image_count = len(_mm_counts.get("image_url", []))
video_count = len(_mm_counts.get("video_url", []))
audio_count = len(_mm_counts.get("audio_url", []))
Expand Down
Loading
Loading