diff --git a/components/src/dynamo/sglang/args.py b/components/src/dynamo/sglang/args.py index 45da91973149..2b019da87342 100644 --- a/components/src/dynamo/sglang/args.py +++ b/components/src/dynamo/sglang/args.py @@ -405,6 +405,10 @@ async def parse_args( # otherwise fall back to default endpoints namespace = dynamo_config.namespace + # Dynamo's parser consumes --enable-multimodal; forward it to SGLang. + if dynamo_config.enable_multimodal: + parsed_args.enable_multimodal = True + # If --embedding-worker is set, also set SGLang's --is-embedding flag if dynamo_config.embedding_worker: parsed_args.is_embedding = True diff --git a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py index be1e1f7954e5..0b63d0e8cb0e 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py @@ -23,6 +23,7 @@ from dynamo.sglang.publisher import DynamoSglangPublisher from dynamo.sglang.request_handlers.handler_base import BaseWorkerHandler from dynamo.sglang.request_handlers.llm.mm_disagg_utils import ( + AUDIO_URL_KEY, IMAGE_URL_KEY, VIDEO_URL_KEY, build_disagg_mm_kwargs, @@ -424,9 +425,10 @@ async def generate( else: raise_if_unextracted_multimodal(request) - # Extract image/video URLs for multimodal requests. SGLang's mm_data_processor + # Extract media URLs for multimodal requests. SGLang's mm_data_processor # handles loading/preprocessing, and the scheduler does vision encoding. mm_data = request.get("multi_modal_data", {}) + audio_data = extract_media_urls(mm_data, AUDIO_URL_KEY) video_data = extract_media_urls(mm_data, VIDEO_URL_KEY) image_data: list[str] | list[PILImage] | None @@ -458,6 +460,7 @@ async def generate( agg = await self.engine.async_generate( **input_param, image_data=image_data, + audio_data=audio_data, video_data=video_data, sampling_params=sampling_params, stream=True, diff --git a/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py b/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py index 71e279d4cac4..275a5d8bf418 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py +++ b/components/src/dynamo/sglang/request_handlers/llm/mm_disagg_utils.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """Multimodal media extraction shared by the disaggregated prefill and decode -workers, so both feed identical image/video URLs to the engine and reproduce the -same token layout the transferred KV depends on. +workers, so both feed identical media URLs to the engine and reproduce the same +token layout the transferred KV depends on. """ import logging @@ -12,8 +12,11 @@ logger = logging.getLogger(__name__) IMAGE_URL_KEY = "image_url" +AUDIO_URL_KEY = "audio_url" VIDEO_URL_KEY = "video_url" -_SUPPORTED_MULTIMODAL_CONTENT_TYPES = frozenset({IMAGE_URL_KEY, VIDEO_URL_KEY}) +_SUPPORTED_MULTIMODAL_CONTENT_TYPES = frozenset( + {IMAGE_URL_KEY, AUDIO_URL_KEY, VIDEO_URL_KEY} +) def _multi_modal_data(request: Dict[str, Any]) -> Dict[str, Any]: @@ -66,8 +69,8 @@ def raise_if_unextracted_multimodal(request: Dict[str, Any]) -> None: message = ( "Multimodal input received but SGLang worker did not receive " f"corresponding multi_modal_data for: {types_str}. Ensure the " - "frontend processor extracted image_url/video_url content or remove " - "image_url/video_url content." + "frontend processor extracted image_url/audio_url/video_url content or " + "remove the corresponding multimodal content." ) logger.error(message) raise RuntimeError(message) @@ -112,20 +115,27 @@ def extract_media_urls( def build_disagg_mm_kwargs(request: Dict[str, Any]) -> Dict[str, Any]: - """Build the ``image_data``/``video_data`` kwargs for a disaggregated worker's - ``async_generate`` call. Both keys are always present (``None`` when absent). + """Build media kwargs for a disaggregated worker's ``async_generate`` call. + + All keys are always present (``None`` when absent). """ mm_data = _multi_modal_data(request) image_data = extract_media_urls(mm_data, IMAGE_URL_KEY) + audio_data = extract_media_urls(mm_data, AUDIO_URL_KEY) video_data = extract_media_urls(mm_data, VIDEO_URL_KEY) # TODO: Native EP/D works with this raw-media path, but both prefill and # decode call it, so SGLang may fetch/load/preprocess the same media twice. # Remove the duplicate preprocessing once native EP/D can share processed # media or embeddings while preserving decode-side token layout. - if image_data or video_data: + if image_data or audio_data or video_data: logger.debug( - "disaggregated multimodal request: images=%d, videos=%d", + "disaggregated multimodal request: images=%d, audio=%d, videos=%d", len(image_data or []), + len(audio_data or []), len(video_data or []), ) - return {"image_data": image_data, "video_data": video_data} + return { + "image_data": image_data, + "audio_data": audio_data, + "video_data": video_data, + } diff --git a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py index 5ad4aefc11b2..74193a180d13 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py +++ b/components/src/dynamo/sglang/tests/test_sglang_decode_handler.py @@ -16,6 +16,7 @@ _user_stop_token_ids, ) from dynamo.sglang.request_handlers.llm.mm_disagg_utils import ( + build_disagg_mm_kwargs, extract_media_urls, raise_if_unextracted_multimodal, ) @@ -53,6 +54,22 @@ def test_extract_media_urls_supports_string_and_wire_items(): ] +def test_build_disagg_mm_kwargs_includes_audio_urls(): + request = { + "multi_modal_data": { + "image_url": [{"Url": "https://example.com/image.png"}], + "audio_url": [{"Url": "https://example.com/audio.wav"}], + "video_url": [{"Url": "https://example.com/video.mp4"}], + } + } + + assert build_disagg_mm_kwargs(request) == { + "image_data": ["https://example.com/image.png"], + "audio_data": ["https://example.com/audio.wav"], + "video_data": ["https://example.com/video.mp4"], + } + + def test_extract_media_urls_returns_none_for_missing_modality(): assert extract_media_urls({}, "image_url") is None assert extract_media_urls(None, "image_url") is None @@ -315,6 +332,25 @@ def test_raises_for_image_url(self, request_factory): with pytest.raises(RuntimeError, match="multi_modal_data"): raise_if_unextracted_multimodal(request_factory(self._image_message())) + def test_raises_for_audio_url(self): + request = { + "token_ids": [1, 2, 3], + "messages": [ + { + "role": "user", + "content": [ + { + "type": "audio_url", + "audio_url": {"url": "https://example.com/audio.wav"}, + } + ], + } + ], + } + + with pytest.raises(RuntimeError, match="audio_url"): + raise_if_unextracted_multimodal(request) + def test_text_only_request_bypasses_guard(self): raise_if_unextracted_multimodal({"token_ids": [10, 20, 30]}) diff --git a/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py b/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py index ef412faa656d..5c295faad2a8 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py +++ b/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py @@ -108,8 +108,8 @@ async def _empty_stream() -> AsyncGenerator[Dict[str, Any], None]: @pytest.mark.asyncio -async def test_aggregated_fd_off_passes_url_strings(): - """Without --frontend-decoding, image_url items pass through as URL strings.""" +async def test_aggregated_fd_off_passes_media_url_strings(): + """Without frontend decoding, media URL items pass through as strings.""" handler = _new_decode_handler(enable_frontend_decoding=False) captured: Dict[str, Any] = {} @@ -122,13 +122,17 @@ async def fake_async_generate(**kwargs): request = { "token_ids": [1, 2, 3], - "multi_modal_data": {"image_url": [{"Url": "https://example.com/a.jpg"}]}, + "multi_modal_data": { + "image_url": [{"Url": "https://example.com/a.jpg"}], + "audio_url": [{"Url": "https://example.com/a.wav"}], + }, } async for _ in handler.generate(request, _Context()): pass assert captured["image_data"] == ["https://example.com/a.jpg"] + assert captured["audio_data"] == ["https://example.com/a.wav"] @pytest.mark.asyncio