Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions components/src/dynamo/sglang/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,10 @@ async def parse_args(
# otherwise fall back to default endpoints
namespace = dynamo_config.namespace

# Dynamo's parser consumes --enable-multimodal; forward it to SGLang.
if dynamo_config.enable_multimodal:
parsed_args.enable_multimodal = True

# If --embedding-worker is set, also set SGLang's --is-embedding flag
if dynamo_config.embedding_worker:
parsed_args.is_embedding = True
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from dynamo.sglang.publisher import DynamoSglangPublisher
from dynamo.sglang.request_handlers.handler_base import BaseWorkerHandler
from dynamo.sglang.request_handlers.llm.mm_disagg_utils import (
AUDIO_URL_KEY,
IMAGE_URL_KEY,
VIDEO_URL_KEY,
build_disagg_mm_kwargs,
Expand Down Expand Up @@ -424,9 +425,10 @@ async def generate(
else:
raise_if_unextracted_multimodal(request)

# Extract image/video URLs for multimodal requests. SGLang's mm_data_processor
# Extract media URLs for multimodal requests. SGLang's mm_data_processor
# handles loading/preprocessing, and the scheduler does vision encoding.
mm_data = request.get("multi_modal_data", {})
audio_data = extract_media_urls(mm_data, AUDIO_URL_KEY)
video_data = extract_media_urls(mm_data, VIDEO_URL_KEY)

image_data: list[str] | list[PILImage] | None
Expand Down Expand Up @@ -458,6 +460,7 @@ async def generate(
agg = await self.engine.async_generate(
**input_param,
image_data=image_data,
audio_data=audio_data,
video_data=video_data,
sampling_params=sampling_params,
stream=True,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@
# SPDX-License-Identifier: Apache-2.0

"""Multimodal media extraction shared by the disaggregated prefill and decode
workers, so both feed identical image/video URLs to the engine and reproduce the
same token layout the transferred KV depends on.
workers, so both feed identical media URLs to the engine and reproduce the same
token layout the transferred KV depends on.
"""

import logging
Expand All @@ -12,8 +12,11 @@
logger = logging.getLogger(__name__)

IMAGE_URL_KEY = "image_url"
AUDIO_URL_KEY = "audio_url"
VIDEO_URL_KEY = "video_url"
_SUPPORTED_MULTIMODAL_CONTENT_TYPES = frozenset({IMAGE_URL_KEY, VIDEO_URL_KEY})
_SUPPORTED_MULTIMODAL_CONTENT_TYPES = frozenset(
{IMAGE_URL_KEY, AUDIO_URL_KEY, VIDEO_URL_KEY}
)


def _multi_modal_data(request: Dict[str, Any]) -> Dict[str, Any]:
Expand Down Expand Up @@ -66,8 +69,8 @@ def raise_if_unextracted_multimodal(request: Dict[str, Any]) -> None:
message = (
"Multimodal input received but SGLang worker did not receive "
f"corresponding multi_modal_data for: {types_str}. Ensure the "
"frontend processor extracted image_url/video_url content or remove "
"image_url/video_url content."
"frontend processor extracted image_url/audio_url/video_url content or "
"remove the corresponding multimodal content."
)
logger.error(message)
raise RuntimeError(message)
Expand Down Expand Up @@ -112,20 +115,27 @@ def extract_media_urls(


def build_disagg_mm_kwargs(request: Dict[str, Any]) -> Dict[str, Any]:
"""Build the ``image_data``/``video_data`` kwargs for a disaggregated worker's
``async_generate`` call. Both keys are always present (``None`` when absent).
"""Build media kwargs for a disaggregated worker's ``async_generate`` call.

All keys are always present (``None`` when absent).
"""
mm_data = _multi_modal_data(request)
image_data = extract_media_urls(mm_data, IMAGE_URL_KEY)
audio_data = extract_media_urls(mm_data, AUDIO_URL_KEY)
video_data = extract_media_urls(mm_data, VIDEO_URL_KEY)
# TODO: Native EP/D works with this raw-media path, but both prefill and
# decode call it, so SGLang may fetch/load/preprocess the same media twice.
# Remove the duplicate preprocessing once native EP/D can share processed
# media or embeddings while preserving decode-side token layout.
if image_data or video_data:
if image_data or audio_data or video_data:
logger.debug(
"disaggregated multimodal request: images=%d, videos=%d",
"disaggregated multimodal request: images=%d, audio=%d, videos=%d",
len(image_data or []),
len(audio_data or []),
len(video_data or []),
)
return {"image_data": image_data, "video_data": video_data}
return {
"image_data": image_data,
"audio_data": audio_data,
"video_data": video_data,
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
_user_stop_token_ids,
)
from dynamo.sglang.request_handlers.llm.mm_disagg_utils import (
build_disagg_mm_kwargs,
extract_media_urls,
raise_if_unextracted_multimodal,
)
Expand Down Expand Up @@ -53,6 +54,22 @@ def test_extract_media_urls_supports_string_and_wire_items():
]


def test_build_disagg_mm_kwargs_includes_audio_urls():
request = {
"multi_modal_data": {
"image_url": [{"Url": "https://example.com/image.png"}],
"audio_url": [{"Url": "https://example.com/audio.wav"}],
"video_url": [{"Url": "https://example.com/video.mp4"}],
}
}

assert build_disagg_mm_kwargs(request) == {
"image_data": ["https://example.com/image.png"],
"audio_data": ["https://example.com/audio.wav"],
"video_data": ["https://example.com/video.mp4"],
}


def test_extract_media_urls_returns_none_for_missing_modality():
assert extract_media_urls({}, "image_url") is None
assert extract_media_urls(None, "image_url") is None
Expand Down Expand Up @@ -315,6 +332,25 @@ def test_raises_for_image_url(self, request_factory):
with pytest.raises(RuntimeError, match="multi_modal_data"):
raise_if_unextracted_multimodal(request_factory(self._image_message()))

def test_raises_for_audio_url(self):
request = {
"token_ids": [1, 2, 3],
"messages": [
{
"role": "user",
"content": [
{
"type": "audio_url",
"audio_url": {"url": "https://example.com/audio.wav"},
}
],
}
],
}

with pytest.raises(RuntimeError, match="audio_url"):
raise_if_unextracted_multimodal(request)

def test_text_only_request_bypasses_guard(self):
raise_if_unextracted_multimodal({"token_ids": [10, 20, 30]})

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,8 +108,8 @@ async def _empty_stream() -> AsyncGenerator[Dict[str, Any], None]:


@pytest.mark.asyncio
async def test_aggregated_fd_off_passes_url_strings():
"""Without --frontend-decoding, image_url items pass through as URL strings."""
async def test_aggregated_fd_off_passes_media_url_strings():
"""Without frontend decoding, media URL items pass through as strings."""
handler = _new_decode_handler(enable_frontend_decoding=False)

captured: Dict[str, Any] = {}
Expand All @@ -122,13 +122,17 @@ async def fake_async_generate(**kwargs):

request = {
"token_ids": [1, 2, 3],
"multi_modal_data": {"image_url": [{"Url": "https://example.com/a.jpg"}]},
"multi_modal_data": {
"image_url": [{"Url": "https://example.com/a.jpg"}],
"audio_url": [{"Url": "https://example.com/a.wav"}],
},
}

async for _ in handler.generate(request, _Context()):
pass

assert captured["image_data"] == ["https://example.com/a.jpg"]
assert captured["audio_data"] == ["https://example.com/a.wav"]


@pytest.mark.asyncio
Expand Down
Loading