diff --git a/components/src/dynamo/common/configuration/groups/frontend_decoding_args.py b/components/src/dynamo/common/configuration/groups/frontend_decoding_args.py new file mode 100644 index 000000000000..8f051583d026 --- /dev/null +++ b/components/src/dynamo/common/configuration/groups/frontend_decoding_args.py @@ -0,0 +1,43 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared `--frontend-decoding` flag helper. + +Both `dynamo.vllm` and `dynamo.sglang` (and any future backend) expose the +same `--frontend-decoding` knob — when set, the Rust frontend decodes +multimodal images and ships pre-decoded pixels via NIXL RDMA instead of +letting the engine fetch + decode. The flag definition is identical +modulo the engine-specific env-var prefix (`DYN_VLLM_*` vs `DYN_SGL_*`), +so the per-backend `backend_args.py` modules call this helper instead of +duplicating the `add_negatable_bool_argument` block. + +Engine-specific validation (e.g. sglang rejects FD combined with the EPD +topology) stays in each backend's `ConfigBase.validate()` — only the +flag plumbing is shared. +""" + +from dynamo.common.configuration.utils import add_negatable_bool_argument + + +def add_frontend_decoding_arg(g, *, env_prefix: str) -> None: + """Add the shared `--frontend-decoding` flag to an argparse group. + + Args: + g: an argparse argument group (from `parser.add_argument_group(...)`). + env_prefix: engine identifier inserted into the env var, e.g. `"VLLM"` + yields `DYN_VLLM_FRONTEND_DECODING`, `"SGL"` yields + `DYN_SGL_FRONTEND_DECODING`. + """ + add_negatable_bool_argument( + g, + flag_name="--frontend-decoding", + env_var=f"DYN_{env_prefix}_FRONTEND_DECODING", + default=False, + help=( + "Enable frontend decoding of multimodal images. " + "Images are decoded in the Rust frontend and transferred to the " + "backend via NIXL RDMA, bypassing in-engine HTTP fetch + decode. " + "Engine-specific topology constraints may apply — see backend " + "validation." + ), + ) diff --git a/components/src/dynamo/sglang/backend_args.py b/components/src/dynamo/sglang/backend_args.py index cd3c70fee99c..199eb3a24636 100644 --- a/components/src/dynamo/sglang/backend_args.py +++ b/components/src/dynamo/sglang/backend_args.py @@ -8,6 +8,9 @@ from dynamo.common.configuration.arg_group import ArgGroup from dynamo.common.configuration.config_base import ConfigBase +from dynamo.common.configuration.groups.frontend_decoding_args import ( + add_frontend_decoding_arg, +) from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument from dynamo.common.constants import EmbeddingTransferMode @@ -109,6 +112,10 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: help="Enable RL training support. Registers the call_tokenizer_manager engine route for generic tokenizer_manager passthrough.", ) + # Topology constraint: rejecting --frontend-decoding combined with the + # EPD multimodal flags happens in DynamoSGLangConfig.validate() below. + add_frontend_decoding_arg(g, env_prefix="SGL") + class DynamoSGLangConfig(ConfigBase): """Configuration for Dynamo SGLang wrapper (SGLang-specific only).""" @@ -125,6 +132,7 @@ class DynamoSGLangConfig(ConfigBase): video_generation_worker: bool enable_rl: bool + frontend_decoding: bool = False def validate(self) -> None: if not isinstance(self.embedding_transfer_mode, EmbeddingTransferMode): @@ -136,3 +144,14 @@ def validate(self) -> None: raise ValueError( "Both 'disagg_config' and 'disagg_config_key' must be provided together." ) + + if self.frontend_decoding and ( + self.multimodal_encode_worker or self.multimodal_worker + ): + raise ValueError( + "--frontend-decoding is incompatible with the EPD multimodal topology " + "(--multimodal-encode-worker / --multimodal-worker). The encode worker " + "needs URLs to run MMEncoder, while --frontend-decoding ships pre-decoded " + "pixels. Use --frontend-decoding on the default aggregated decode worker " + "(no --multimodal-* flag) instead." + ) diff --git a/components/src/dynamo/sglang/init_llm.py b/components/src/dynamo/sglang/init_llm.py index 4e9b923c34dc..1655b494da7f 100644 --- a/components/src/dynamo/sglang/init_llm.py +++ b/components/src/dynamo/sglang/init_llm.py @@ -119,7 +119,12 @@ async def init_decode( ready_event = asyncio.Event() handler = DecodeWorkerHandler( - engine, config, publisher, generate_endpoint, shutdown_event + engine, + config, + publisher, + generate_endpoint, + shutdown_event, + enable_frontend_decoding=dynamo_args.frontend_decoding, ) handler.register_engine_routes(runtime) diff --git a/components/src/dynamo/sglang/register.py b/components/src/dynamo/sglang/register.py index 91114bd9d84f..ae1ea4114d04 100644 --- a/components/src/dynamo/sglang/register.py +++ b/components/src/dynamo/sglang/register.py @@ -4,6 +4,7 @@ import asyncio import json import logging +import os from typing import Any, List, Optional import sglang as sgl @@ -14,13 +15,37 @@ from dynamo._core import Endpoint from dynamo.common.utils.output_modalities import get_output_modalities -from dynamo.llm import ModelInput, ModelRuntimeConfig, ModelType, register_model +from dynamo.llm import ( + MediaDecoder, + MediaFetcher, + ModelInput, + ModelRuntimeConfig, + ModelType, + register_model, +) from dynamo.sglang._compat import get_scheduler_info from dynamo.sglang.args import DynamoConfig SGLANG_HICACHE_MOONCAKE_RUNTIME_KEY = "sglang_hicache_mooncake" +def _build_media_decoder_and_fetcher(): + """Construct MediaDecoder/MediaFetcher for frontend-decoded multimodal. + + Mirrors the vLLM backend pattern (components/src/dynamo/vllm/main.py). + """ + media_decoder = MediaDecoder() + media_decoder.enable_image({"limits": {"max_alloc": 128 * 1024 * 1024}}) + + media_fetcher = MediaFetcher() + media_fetcher.timeout_ms(30000) + allow_internal = os.getenv("DYN_MM_ALLOW_INTERNAL", "0") == "1" + media_fetcher.allow_direct_ip(allow_internal) + media_fetcher.allow_direct_port(allow_internal) + + return media_decoder, media_fetcher + + async def _register_model_with_runtime_config( engine: sgl.Engine, endpoint: Endpoint, @@ -53,6 +78,13 @@ async def _register_model_with_runtime_config( if output_type != ModelType.Embedding: output_type = ModelType.Chat + # Configure the Rust frontend's media decoder so it ships pre-decoded + # images via NIXL RDMA instead of forwarding raw URLs / base64 to us. + media_decoder = None + media_fetcher = None + if getattr(dynamo_args, "frontend_decoding", False): + media_decoder, media_fetcher = _build_media_decoder_and_fetcher() + try: await register_model( input_type, @@ -64,6 +96,8 @@ async def _register_model_with_runtime_config( kv_cache_block_size=server_args.page_size, runtime_config=runtime_config, custom_template_path=dynamo_args.custom_jinja_template, + media_decoder=media_decoder, + media_fetcher=media_fetcher, ) logging.info("Successfully registered LLM with runtime config") return True diff --git a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py index 55d9748e8ec9..b65445af62cc 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py @@ -9,9 +9,11 @@ import pybase64 import sglang as sgl +from PIL.Image import Image as PILImage from dynamo._core import Context from dynamo.common.constants import DisaggregationMode +from dynamo.common.multimodal.image_loader import ImageLoader from dynamo.common.utils.engine_response import normalize_finish_reason from dynamo.common.utils.otel_tracing import build_trace_headers from dynamo.sglang._compat import filter_supported_async_generate_kwargs @@ -155,6 +157,7 @@ def __init__( publisher: Optional[DynamoSglangPublisher] = None, generate_endpoint=None, shutdown_event: Optional[asyncio.Event] = None, + enable_frontend_decoding: bool = False, ) -> None: """Initialize decode worker handler. @@ -164,6 +167,10 @@ def __init__( publisher: Metrics publisher for the worker. shutdown_event: Optional event to signal shutdown. generate_endpoint: The endpoint handle for discovery registration. + enable_frontend_decoding: If True, multimodal images arrive as + ``Decoded`` variants over NIXL RDMA from the Rust frontend + and must be read+converted to PIL before passing to SGLang. + Off by default; the worker keeps the URL-string fast path. """ super().__init__( engine, @@ -180,12 +187,18 @@ def __init__( self._routed_experts_kwargs: Dict[ str, Any ] = self._resolve_routed_experts_kwargs(self.engine, self.config.server_args) + self._enable_frontend_decoding = enable_frontend_decoding + self._image_loader: Optional[ImageLoader] = None + if self._enable_frontend_decoding: + # Lazy-inits a NIXL connector internally for Decoded variants. + self._image_loader = ImageLoader(enable_frontend_decoding=True) if self.serving_mode == DisaggregationMode.DECODE: logging.info( "Decode worker handler initialized (disaggregated decode mode)" ) else: - logging.info("Decode worker handler initialized (aggregated mode)") + mode = "frontend-decoded" if self._enable_frontend_decoding else "standard" + logging.info(f"Decode worker handler initialized (aggregated mode, {mode})") @staticmethod def _resolve_routed_experts_kwargs(engine: Any, server_args: Any) -> Dict[str, Any]: @@ -492,9 +505,22 @@ async def generate( # Extract image/video URLs for multimodal requests. SGLang's mm_data_processor # handles loading/preprocessing, and the scheduler does vision encoding. mm_data = request.get("multi_modal_data", {}) - image_data = _extract_media_urls(mm_data, "image_url") video_data = _extract_media_urls(mm_data, "video_url") + image_data: list[str] | list[PILImage] | None + if self._enable_frontend_decoding: + # Invariant from __init__: _image_loader is non-None iff + # _enable_frontend_decoding is True. Assert narrows the + # Optional for the type checker without runtime branching. + assert self._image_loader is not None + image_items = mm_data.get("image_url") or [] + if image_items: + image_data = await self._image_loader.load_image_batch(image_items) + else: + image_data = None + else: + image_data = _extract_media_urls(mm_data, "image_url") + trace_header = build_trace_headers(context) if self.enable_trace else None # Extract dp_rank from routing info (set by KV router) diff --git a/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py b/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py new file mode 100644 index 000000000000..c14b3f5cdd25 --- /dev/null +++ b/components/src/dynamo/sglang/tests/test_sglang_frontend_decoding.py @@ -0,0 +1,200 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from contextlib import asynccontextmanager +from types import SimpleNamespace +from typing import Any, AsyncGenerator, Dict +from unittest.mock import AsyncMock + +import pytest +from PIL import Image + +from dynamo.common.constants import DisaggregationMode, EmbeddingTransferMode +from dynamo.sglang.backend_args import DynamoSGLangConfig +from dynamo.sglang.request_handlers.llm.decode_handler import DecodeWorkerHandler + +pytestmark = [ + pytest.mark.unit, + pytest.mark.sglang, + pytest.mark.gpu_0, + pytest.mark.profiled_vram_gib(0), + pytest.mark.pre_merge, +] + + +def _make_config( + *, + frontend_decoding: bool = False, + multimodal_encode_worker: bool = False, + multimodal_worker: bool = False, +) -> DynamoSGLangConfig: + # ConfigBase has no kwargs __init__; sibling tests (test_backend_args.py) + # construct via no-arg + setattr. + config = DynamoSGLangConfig() + config.use_sglang_tokenizer = False + config.multimodal_encode_worker = multimodal_encode_worker + config.multimodal_worker = multimodal_worker + config.embedding_transfer_mode = EmbeddingTransferMode.NIXL_WRITE + config.embedding_worker = False + config.image_diffusion_worker = False + config.video_generation_worker = False + config.enable_rl = False + config.frontend_decoding = frontend_decoding + return config + + +def test_validate_rejects_frontend_decoding_with_encode_worker(): + config = _make_config(frontend_decoding=True, multimodal_encode_worker=True) + with pytest.raises(ValueError, match="--frontend-decoding is incompatible"): + config.validate() + + +def test_validate_rejects_frontend_decoding_with_multimodal_worker(): + config = _make_config(frontend_decoding=True, multimodal_worker=True) + with pytest.raises(ValueError, match="--frontend-decoding is incompatible"): + config.validate() + + +def test_validate_accepts_frontend_decoding_alone(): + config = _make_config(frontend_decoding=True) + config.validate() + + +class _Context: + id_value: str = "test-request" + trace_id: str = "test-trace" + + def id(self) -> str: + return self.id_value + + def is_stopped(self) -> bool: + return False + + +def _new_decode_handler(*, enable_frontend_decoding: bool): + """Build a DecodeWorkerHandler without invoking sgl.Engine. + + Mirrors the pattern in test_sglang_decode_handler.py — bypass __init__ and + manually set the few attributes the methods we exercise actually read. + """ + handler = DecodeWorkerHandler.__new__(DecodeWorkerHandler) + handler.use_sglang_tokenizer = False + handler.enable_trace = False + handler.serving_mode = DisaggregationMode.AGGREGATED + handler.config = SimpleNamespace( + server_args=SimpleNamespace(served_model_name="test-model") + ) + handler._routed_experts_kwargs = {} + handler._enable_frontend_decoding = enable_frontend_decoding + handler._image_loader = None + + @asynccontextmanager + async def no_cancellation_monitor(*args, **kwargs): + yield None + + handler._cancellation_monitor = no_cancellation_monitor + + handler._get_input_param = lambda req: {"input_ids": req.get("token_ids", [])} + handler._resolve_lora = lambda req: None + handler._session_kwargs = lambda req: {} + handler._priority_kwargs = lambda priority: {} + + return handler + + +async def _empty_stream() -> AsyncGenerator[Dict[str, Any], None]: + if False: # pragma: no cover — never yields + yield {} + + +@pytest.mark.asyncio +async def test_aggregated_fd_off_passes_url_strings(): + """Without --frontend-decoding, image_url items pass through as URL strings.""" + handler = _new_decode_handler(enable_frontend_decoding=False) + + captured: Dict[str, Any] = {} + + async def fake_async_generate(**kwargs): + captured.update(kwargs) + return _empty_stream() + + handler.engine = SimpleNamespace(async_generate=fake_async_generate) + + request = { + "token_ids": [1, 2, 3], + "multi_modal_data": {"image_url": [{"Url": "https://example.com/a.jpg"}]}, + } + + async for _ in handler.generate(request, _Context()): + pass + + assert captured["image_data"] == ["https://example.com/a.jpg"] + + +@pytest.mark.asyncio +async def test_aggregated_fd_on_loads_decoded_variants_to_pil(): + """With --frontend-decoding, Decoded items are loaded via ImageLoader and + forwarded as PIL Images (not strings) to engine.async_generate.""" + handler = _new_decode_handler(enable_frontend_decoding=True) + + decoded_metadata = { + "shape": [4, 4, 3], + "dtype": "uint8", + "agent_metadata": "stub", + "remote_descriptor": "stub", + } + pil_stub = Image.new("RGB", (4, 4), (255, 0, 0)) + + image_loader = SimpleNamespace( + load_image_batch=AsyncMock(return_value=[pil_stub]), + ) + handler._image_loader = image_loader + + captured: Dict[str, Any] = {} + + async def fake_async_generate(**kwargs): + captured.update(kwargs) + return _empty_stream() + + handler.engine = SimpleNamespace(async_generate=fake_async_generate) + + request = { + "token_ids": [1, 2, 3], + "multi_modal_data": {"image_url": [{"Decoded": decoded_metadata}]}, + } + + async for _ in handler.generate(request, _Context()): + pass + + image_loader.load_image_batch.assert_awaited_once_with( + [{"Decoded": decoded_metadata}] + ) + assert captured["image_data"] == [pil_stub] + + +@pytest.mark.asyncio +async def test_aggregated_fd_on_no_images_passes_none(): + """FD on, but request has no images — image_data must be None (not []).""" + handler = _new_decode_handler(enable_frontend_decoding=True) + handler._image_loader = SimpleNamespace( + load_image_batch=AsyncMock( + side_effect=AssertionError( + "load_image_batch must not run when there are no images" + ) + ), + ) + + captured: Dict[str, Any] = {} + + async def fake_async_generate(**kwargs): + captured.update(kwargs) + return _empty_stream() + + handler.engine = SimpleNamespace(async_generate=fake_async_generate) + + request = {"token_ids": [1, 2, 3], "multi_modal_data": {}} + + async for _ in handler.generate(request, _Context()): + pass + + assert captured["image_data"] is None diff --git a/components/src/dynamo/trtllm/backend_args.py b/components/src/dynamo/trtllm/backend_args.py index daf38a27beb2..8c345e78dba7 100644 --- a/components/src/dynamo/trtllm/backend_args.py +++ b/components/src/dynamo/trtllm/backend_args.py @@ -10,6 +10,9 @@ from dynamo.common.configuration.arg_group import ArgGroup from dynamo.common.configuration.config_base import ConfigBase +from dynamo.common.configuration.groups.frontend_decoding_args import ( + add_frontend_decoding_arg, +) from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument from . import __version__ @@ -208,17 +211,7 @@ def add_arguments(self, parser: argparse.ArgumentParser) -> None: arg_type=int, help="Maximum size of downloadable embedding files/Image URLs.", ) - add_negatable_bool_argument( - g, - flag_name="--frontend-decoding", - env_var="DYN_TRTLLM_FRONTEND_DECODING", - default=False, - help=( - "Enable frontend decoding of multimodal images. " - "When enabled, images are decoded in the Rust frontend and transferred to the backend via NIXL RDMA. " - "Without this flag, images are decoded in the Python backend (default behavior)." - ), - ) + add_frontend_decoding_arg(g, env_prefix="TRTLLM") # --- Guided Decoding --- add_argument( diff --git a/components/src/dynamo/trtllm/workers/llm_worker.py b/components/src/dynamo/trtllm/workers/llm_worker.py index 9e0698520c76..57470c18bf15 100644 --- a/components/src/dynamo/trtllm/workers/llm_worker.py +++ b/components/src/dynamo/trtllm/workers/llm_worker.py @@ -45,6 +45,8 @@ from dynamo.common.utils.runtime import parse_endpoint from dynamo.llm import ( KvEventPublisher, + MediaDecoder, + MediaFetcher, ModelInput, ModelRuntimeConfig, ModelType, @@ -63,18 +65,6 @@ ) from dynamo.trtllm.utils.trtllm_utils import deep_update -# Optional imports for Rust frontend media decoding support -MediaDecoder: type | None = None -MediaFetcher: type | None = None -try: - from dynamo.llm import MediaDecoder, MediaFetcher - - MEDIA_DECODER_AVAILABLE = True -except ImportError: - MediaDecoder = None - MediaFetcher = None - MEDIA_DECODER_AVAILABLE = False - # Default buffer size for kv cache events. DEFAULT_KV_EVENT_BUFFER_MAX_SIZE = 1024 @@ -617,12 +607,6 @@ async def init_llm_worker( media_decoder = None media_fetcher = None if config.frontend_decoding: - if not MEDIA_DECODER_AVAILABLE: - raise RuntimeError( - "--frontend-decoding requires MediaDecoder support. " - "Ensure dynamo.llm module includes MediaDecoder and MediaFetcher." - ) - assert MediaDecoder is not None and MediaFetcher is not None media_decoder = MediaDecoder() media_decoder.enable_image({"limits": {"max_alloc": 128 * 1024 * 1024}}) media_fetcher = MediaFetcher() diff --git a/components/src/dynamo/vllm/backend_args.py b/components/src/dynamo/vllm/backend_args.py index 28c9d0eb3034..b44d2adeef79 100644 --- a/components/src/dynamo/vllm/backend_args.py +++ b/components/src/dynamo/vllm/backend_args.py @@ -8,6 +8,9 @@ from dynamo.common.configuration.arg_group import ArgGroup from dynamo.common.configuration.config_base import ConfigBase +from dynamo.common.configuration.groups.frontend_decoding_args import ( + add_frontend_decoding_arg, +) from dynamo.common.configuration.utils import add_argument, add_negatable_bool_argument from . import __version__ @@ -114,17 +117,7 @@ def add_arguments(self, parser) -> None: ), ) - add_negatable_bool_argument( - g, - flag_name="--frontend-decoding", - env_var="DYN_VLLM_FRONTEND_DECODING", - default=False, - help=( - "Enable frontend decoding of multimodal images. " - "When enabled, images are decoded in the Rust frontend and transferred to the backend via NIXL RDMA. " - "Without this flag, images are decoded in the Python backend (default behavior)." - ), - ) + add_frontend_decoding_arg(g, env_prefix="VLLM") add_argument( g, diff --git a/components/src/dynamo/vllm/main.py b/components/src/dynamo/vllm/main.py index f05dc9d22714..e089973bdb6e 100644 --- a/components/src/dynamo/vllm/main.py +++ b/components/src/dynamo/vllm/main.py @@ -29,6 +29,8 @@ from dynamo.common.utils.runtime import create_runtime from dynamo.llm import ( KvEventPublisher, + MediaDecoder, + MediaFetcher, ModelInput, ModelRuntimeConfig, ModelType, @@ -47,18 +49,6 @@ from .publisher import DYNAMO_COMPONENT_REGISTRY, StatLoggerFactory from .snapshot import prepare_snapshot_engine -# Optional imports for frontend decoding support -MediaDecoder: type | None = None -MediaFetcher: type | None = None -try: - from dynamo.llm import MediaDecoder, MediaFetcher - - MEDIA_DECODER_AVAILABLE = True -except ImportError: - MediaDecoder = None - MediaFetcher = None - MEDIA_DECODER_AVAILABLE = False - configure_dynamo_logging() logger = logging.getLogger(__name__) shutdown_endpoints: list = [] @@ -671,12 +661,6 @@ async def register_vllm_model( media_decoder = None media_fetcher = None if config.frontend_decoding: - if not MEDIA_DECODER_AVAILABLE: - raise RuntimeError( - "--frontend-decoding requires MediaDecoder support. " - "Ensure dynamo.llm module includes MediaDecoder and MediaFetcher." - ) - assert MediaDecoder is not None and MediaFetcher is not None media_decoder = MediaDecoder() media_decoder.enable_image({"limits": {"max_alloc": 128 * 1024 * 1024}}) # media_decoder.enable_video({}) diff --git a/examples/backends/sglang/launch/agg_vision.sh b/examples/backends/sglang/launch/agg_vision.sh index 64e65aa8b1c0..ff4cd3a795d3 100755 --- a/examples/backends/sglang/launch/agg_vision.sh +++ b/examples/backends/sglang/launch/agg_vision.sh @@ -3,6 +3,9 @@ # SPDX-License-Identifier: Apache-2.0 # # Aggregated multimodal (image/video + LLM) serving. +# Pass --frontend-decoding to route the image through the Rust frontend's +# NIXL RDMA pipeline (decoded pixels → SGLang worker as PIL Images) instead +# of letting SGLang fetch+decode internally. # GPUs: 1 set -e @@ -13,10 +16,13 @@ source "$SCRIPT_DIR/../../../common/gpu_utils.sh" # build_sglang_gpu_mem_args source "$SCRIPT_DIR/../../../common/launch_utils.sh" # Default values -# TODO: Update default to Qwen3-VL-2B-Instruct after SGLang 0.5.10+ upgrade. -MODEL="Qwen/Qwen2-VL-7B-Instruct" +MODEL="Qwen/Qwen3-VL-2B-Instruct" CHAT_TEMPLATE="" +# SGLang KV-cache page size. Default 16; must be 1 for hybrid Mamba models +# (e.g. Qwen3.5-0.8B) whose MambaRadixCache asserts page_size == 1. +PAGE_SIZE=16 ENABLE_OTEL=false +FRONTEND_DECODING=false # Parse command line arguments EXTRA_ARGS=() @@ -30,16 +36,28 @@ while [[ $# -gt 0 ]]; do CHAT_TEMPLATE="$2" shift 2 ;; + --page-size) + PAGE_SIZE="$2" + shift 2 + ;; --enable-otel) ENABLE_OTEL=true shift ;; + --frontend-decoding) + FRONTEND_DECODING=true + shift + ;; -h|--help) echo "Usage: $0 [OPTIONS]" echo "Options:" echo " --model-path Specify model (default: $MODEL)" echo " --chat-template Specify SGLang chat template (default: $CHAT_TEMPLATE)" + echo " --page-size SGLang KV-cache page size (default: $PAGE_SIZE; must be 1 for Mamba)" echo " --enable-otel Enable OpenTelemetry tracing" + echo " --frontend-decoding Decode images in the Rust frontend and" + echo " ship pixels via NIXL RDMA (bypasses" + echo " SGLang's internal HTTP fetch + base64 decode)" echo " -h, --help Show this help message" echo "" echo "Additional SGLang/Dynamo flags can be passed and will be forwarded" @@ -69,10 +87,32 @@ HTTP_PORT="${DYN_HTTP_PORT:-8000}" # direct invocations behave identically to before this hook was added. GPU_MEM_ARGS=$(build_sglang_gpu_mem_args) -print_launch_banner --multimodal "Launching Aggregated Vision Serving" "$MODEL" "$HTTP_PORT" +FD_ARGS=() +BANNER_SUFFIX="" +if [ "$FRONTEND_DECODING" = true ]; then + FD_ARGS+=(--frontend-decoding) + BANNER_SUFFIX=" (Frontend Decoding)" + + # The SGLang dev image inherits NIXL via the nixl_cu12 wheel but does NOT + # add its native libs to LD_LIBRARY_PATH (cf. container/templates/dev.Dockerfile: + # "SGLang dev/local-dev inherit the upstream SGLang/NIXL runtime stack"). + # Without this, dynamo.frontend's Rust runtime hits "NIXL is not supported in + # stub mode" the moment it tries to build a media-fetching pipeline. Point the + # loader at the wheel-shipped .so files explicitly. + NIXL_WHEEL_LIBS="$(python3 -c 'import nixl_cu12, os; print(os.path.join(os.path.dirname(os.path.dirname(nixl_cu12.__file__)), ".nixl_cu12.mesonpy.libs"))' 2>/dev/null || true)" + if [ -d "$NIXL_WHEEL_LIBS" ]; then + export LD_LIBRARY_PATH="${NIXL_WHEEL_LIBS}:${NIXL_WHEEL_LIBS}/plugins:${LD_LIBRARY_PATH}" + export NIXL_PLUGIN_DIR="${NIXL_PLUGIN_DIR:-$NIXL_WHEEL_LIBS/plugins}" + fi +fi + +print_launch_banner --multimodal "Launching Aggregated Vision Serving${BANNER_SUFFIX}" "$MODEL" "$HTTP_PORT" # run ingress # dynamo.frontend accepts either --http-port flag or DYN_HTTP_PORT env var (defaults to 8000) +# Frontend has no --frontend-decoding flag — decoding is opt-in via the +# backend's model card (MediaDecoder configured at register_model time when +# the backend is launched with --frontend-decoding). OTEL_SERVICE_NAME=dynamo-frontend \ python3 -m dynamo.frontend & @@ -82,18 +122,21 @@ if [ -n "$CHAT_TEMPLATE" ]; then TEMPLATE_ARGS+=(--chat-template "$CHAT_TEMPLATE") fi -# run worker with a vision model (SGLang auto-detects chat template from HF tokenizer) -# The SGLang engine handles image/video loading and vision encoding internally. +# run worker with a vision model (SGLang auto-detects chat template from HF tokenizer). +# Without --frontend-decoding, the SGLang engine handles image/video loading and +# vision encoding internally. With it, the worker consumes Decoded items via +# ImageLoader and hands PIL Images to sgl.Engine.async_generate(image_data=[...]). OTEL_SERVICE_NAME=dynamo-worker DYN_SYSTEM_PORT=${DYN_SYSTEM_PORT:-8081} \ python3 -m dynamo.sglang \ --model-path "$MODEL" \ --served-model-name "$MODEL" \ "${TEMPLATE_ARGS[@]}" \ - --page-size 16 \ + --page-size "$PAGE_SIZE" \ --tp 1 \ --trust-remote-code \ --skip-tokenizer-init \ --enable-metrics \ + "${FD_ARGS[@]}" \ $GPU_MEM_ARGS \ "${TRACE_ARGS[@]}" \ "${EXTRA_ARGS[@]}" & diff --git a/tests/serve/test_sglang.py b/tests/serve/test_sglang.py index 7644367c57e2..3634535d1401 100644 --- a/tests/serve/test_sglang.py +++ b/tests/serve/test_sglang.py @@ -18,6 +18,7 @@ from tests.serve.lora_utils import DEFAULT_LORA_REPO, MinioLoraConfig from tests.utils.constants import DefaultPort from tests.utils.engine_process import EngineConfig +from tests.utils.multimodal import make_image_payload_b64 from tests.utils.payload_builder import ( anthropic_messages_payload_default, anthropic_messages_stream_payload_default, @@ -318,6 +319,50 @@ class SGLangConfig(EngineConfig): ) ], ), + "multimodal_agg_fd_qwen": SGLangConfig( + # Aggregated multimodal with --frontend-decoding: the Rust frontend + # decodes the image and ships pre-decoded pixels via NIXL RDMA; + # the SGLang worker consumes Decoded items through ImageLoader and + # hands PIL Images to sgl.Engine.async_generate(image_data=...). + # Mirrors the vLLM FD pattern at + # tests/serve/multimodal_profiles/vllm.py (Qwen3.5-0.8B "b64_frontend_decoding"). + name="multimodal_agg_fd_qwen", + directory=sglang_dir, + script_name="agg_vision.sh", + marks=[ + pytest.mark.gpu_1, + pytest.mark.profiled_vram_gib(4.7), # parity with vLLM Qwen3.5-0.8B + # 4096 covers the b64 PNG image-token expansion (~2198 tokens + # for our 1999x1125 LFS test asset under Qwen3.5-0.8B's vision + # processor) + 100-token max response + headroom. + # TODO: bisect via tests/utils/profile_pytest.py for a tighter bound. + pytest.mark.requested_sglang_kv_tokens(4096), + pytest.mark.timeout(300), + # post_merge: NIXL stubs outside docker can lack the Decoded + # transport path. Same gating as vLLM's FD case + # (tests/serve/multimodal_profiles/vllm.py:67-70). + pytest.mark.post_merge, + ], + model="Qwen/Qwen3.5-0.8B", + script_args=[ + "--model-path", + "Qwen/Qwen3.5-0.8B", + # Hybrid Mamba/full-attention VL: SGLang's MambaRadixCache v1 + # asserts page_size == 1. The launch script defaults to 16. + "--page-size", + "1", + "--frontend-decoding", + ], + delayed_start=0, + timeout=360, + frontend_port=DefaultPort.FRONTEND.value, + request_payloads=[ + # Inline-base64 PNG: exercises strip_inline_data_urls in the + # Rust frontend + NIXL RDMA transfer of decoded pixels — the + # path that distinguishes FD from the plain URL path. + make_image_payload_b64(["green"]), + ], + ), "multimodal_agg_qwen": SGLangConfig( # Tests single-process aggregated multimodal inference using DecodeWorkerHandler # with in-process vision encoding (no separate encode worker)