diff --git a/components/src/dynamo/sglang/AGENTS.md b/components/src/dynamo/sglang/AGENTS.md
index d35048a72096..b7139517a4e0 100644
--- a/components/src/dynamo/sglang/AGENTS.md
+++ b/components/src/dynamo/sglang/AGENTS.md
@@ -70,10 +70,11 @@ Worker dispatch (main.py:60-132):
have `max_running_requests`, `dllm_algorithm_config`, or other LLM-specific fields.
Use `getattr()` when accessing fields that may not exist on the stub.
-SGLang 0.5.17 makes a resolved `ServerArgs` unconditionally read-only. Apply Dynamo's
-post-resolution startup overrides through `_compat.override_server_args()`; control-plane
-updates after engine creation should use the tokenizer manager's update API instead of
-assigning fields on `server_args`.
+The supported SGLang 0.5.18/0.5.19 releases keep raw input on `ServerArgs` and
+publish the resolved configuration separately. Apply Dynamo's post-resolution startup
+overrides through `_compat.override_server_args()`; control-plane updates after engine
+creation should use the tokenizer manager's update API instead of assigning fields on
+`server_args`.
**DynamoConfig** combines `DynamoRuntimeConfig` (common flags like `--namespace`,
`--output-modalities`, `--media-output-fs-url`) with `DynamoSGLangConfig` (sglang-specific
diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py
index a1edb3f39db9..090e90ed5937 100644
--- a/components/src/dynamo/sglang/_compat.py
+++ b/components/src/dynamo/sglang/_compat.py
@@ -20,23 +20,22 @@
>= 0.5.11. Pass through; do not re-encode.
"""
+import importlib
import inspect
import logging
+import uuid
from collections.abc import Mapping
from functools import lru_cache, wraps
+from types import ModuleType
from typing import Any
try:
- from sglang.srt.arg_groups.overrides import declare_late_resolution
-except ImportError:
- # SGLang 0.5.17 and the XPU 0.5.11 pin predate declarations.
- declare_late_resolution = None
-
-try:
- from sglang.srt.arg_groups.overrides import resolved_view as sglang_resolved_view
-except ImportError:
- # SGLang #36255 exposes ServerArgs._resolved() instead.
- sglang_resolved_view = None
+ from sglang.srt.utils.server_args_config_parser import ConfigArgumentMerger
+except ModuleNotFoundError as exc:
+ if exc.name != "sglang.srt.utils.server_args_config_parser":
+ raise
+ # Keep the CUDA 0.5.18 and XPU 0.5.11 pins working until both move here.
+ from sglang.srt.server_args_config_parser import ConfigArgumentMerger
try:
from sglang.srt.arg_groups.overrides import (
@@ -56,15 +55,12 @@
# Remove when min supported version has the accessor move (sgl #36972).
sglang_use_mla_backend = None
-logger = logging.getLogger(__name__)
-
try:
- from sglang.srt.utils.server_args_config_parser import ConfigArgumentMerger
-except ModuleNotFoundError as exc:
- if exc.name != "sglang.srt.utils.server_args_config_parser":
- raise
- # Keep the CUDA 0.5.18 and XPU 0.5.11 pins working until both move here.
- from sglang.srt.server_args_config_parser import ConfigArgumentMerger
+ from sglang.srt.runtime_context import publish as _sglang_publish
+except ImportError:
+ # Fallback for SGLang 0.5.18 and the XPU 0.5.11 pin. Remove the 0.5.18
+ # portion when minimum supported SGLang is 0.5.19+.
+ _sglang_publish = None
def get_sglang_model_config(server_args: Any) -> Any:
@@ -97,6 +93,107 @@ def sglang_uses_mla_backend(server_args: Any) -> bool:
return bool(sglang_use_mla_backend(server_args))
+def publish_server_args(server_args: Any, *, role: str) -> None:
+ """Publish process-wide SGLang configuration when the API is available."""
+ if _sglang_publish is not None:
+ _sglang_publish(server_args, role=role)
+
+
+try:
+ from sglang.srt.arg_groups.overrides import declare_late_resolution
+except ImportError:
+ # The separately pinned XPU SGLang 0.5.11 predates declarations. Remove
+ # when the XPU SGLang pin is upgraded to 0.5.18+.
+ declare_late_resolution = None
+
+try:
+ from sglang.srt.arg_groups.model_override_base import (
+ resolved_view as sglang_resolved_view,
+ )
+except ImportError:
+ # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is 0.5.19+.
+ try:
+ from sglang.srt.arg_groups.overrides import (
+ resolved_view as sglang_resolved_view,
+ )
+ except ImportError:
+ # The separately pinned XPU SGLang 0.5.11 stores effective values on
+ # ServerArgs directly. Remove when that pin is upgraded.
+ sglang_resolved_view = None
+
+logger = logging.getLogger(__name__)
+
+
+def get_mm_encoder_class() -> type[Any]:
+ """Load MMEncoder from the supported SGLang package layout.
+
+ Keep this import deferred because the encoder module imports compiled CUDA
+ operators and this compatibility module is also collected on CPU-only CI
+ hosts.
+ """
+ try:
+ from sglang.srt.disaggregation.encoder.server import MMEncoder
+ except ImportError:
+ # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is
+ # 0.5.19+.
+ from sglang.srt.disaggregation.encode_server import MMEncoder
+
+ return MMEncoder
+
+
+def get_encoder_preprocessor_modules() -> tuple[ModuleType, ...]:
+ """Return importable encoder modules that bind video preprocessing APIs."""
+ modules: list[ModuleType] = []
+ for module_path in (
+ "sglang.srt.disaggregation.encoder.preprocessor",
+ # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is
+ # 0.5.19+.
+ "sglang.srt.disaggregation.encode_server",
+ ):
+ try:
+ modules.append(importlib.import_module(module_path))
+ except (ImportError, OSError):
+ continue
+ return tuple(modules)
+
+
+async def mm_encode(
+ encoder: Any, media_inputs: list[Any], modality: Any
+) -> tuple[Any, Any, dict[str, Any]]:
+ """Encode media across the supported SGLang MMEncoder APIs."""
+ legacy_encode = getattr(encoder, "_encode", None)
+ if callable(legacy_encode):
+ # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is
+ # 0.5.19+.
+ return await legacy_encode(media_inputs, modality)
+
+ prepare = getattr(encoder, "_prepare_encode_context", None)
+ compute = getattr(encoder, "_compute_embedding", None)
+ if not callable(prepare) or not callable(compute):
+ raise RuntimeError("SGLang MMEncoder does not expose an encode API")
+
+ request = {
+ "req_id": f"dynamo-direct-{uuid.uuid4()}",
+ "num_parts": 1,
+ "part_idx": 0,
+ "mm_items": media_inputs,
+ "hashes": None,
+ }
+ encode_context = await prepare(
+ [request],
+ modality,
+ use_global_cache=False,
+ )
+ embeddings = await compute(encode_context, keep_on_gpu=False)
+ if embeddings is None:
+ raise RuntimeError("SGLang MMEncoder returned no embeddings")
+ return (
+ encode_context.preprocess_result.grid_thw,
+ embeddings,
+ encode_context.aux_data,
+ )
+
+
@lru_cache(maxsize=1)
def _warn_require_reasoning_unsupported() -> None:
logger.warning(
@@ -109,9 +206,10 @@ def _warn_require_reasoning_unsupported() -> None:
def ensure_sglang_tensor_image_size() -> None:
"""Allow SGLang's image-token resolver to handle decoded image tensors.
- SGLang 0.5.13 through 0.5.18 assume every decoded image exposes the PIL
- ``height``/``width`` attributes. Its CUDA JPEG decoder instead returns a
- CHW tensor, causing multimodal requests to fall back to retokenization.
+ SGLang 0.5.13 through the 0.5.19 release branch assume every decoded image
+ exposes the PIL ``height``/``width`` attributes. Its CUDA JPEG decoder
+ instead returns a CHW tensor, causing multimodal requests to fall back to
+ retokenization.
Remove this compatibility override once the minimum supported SGLang
release handles tensor image dimensions itself.
@@ -154,26 +252,14 @@ def override_server_args(server_args: Any, source: str, **fields: Any) -> None:
SGLang 0.5.18+ resolves its effective configuration separately from raw
``ServerArgs`` input. Declare pre-engine changes through its resolution API
- so the engine's resolved projection observes them. SGLang 0.5.17 exposes
- ``ServerArgs.override`` instead. The separately pinned XPU image still uses
- SGLang 0.5.11, which predates both APIs; preserve its legacy assignment
- behavior until its engine pin is upgraded.
+ so the engine's resolved projection observes them. The separately pinned
+ XPU image still uses SGLang 0.5.11, which predates that API; preserve its
+ legacy assignment behavior until its engine pin is upgraded.
"""
if declare_late_resolution is not None:
declare_late_resolution(server_args, source, **fields)
return
- late_resolution = getattr(server_args, "_late_resolution", None)
- if callable(late_resolution):
- late_resolution(source, **fields)
- return
-
- # Fallback for SGLang 0.5.17. Remove when minimum supported SGLang is 0.5.18+.
- override = getattr(server_args, "override", None)
- if callable(override):
- override(source, **fields)
- return
-
# XPU compatibility for SGLang 0.5.11. Remove when the XPU SGLang pin is
# upgraded to 0.5.16+.
for name, value in fields.items():
@@ -183,14 +269,11 @@ def override_server_args(server_args: Any, source: str, **fields: Any) -> None:
def resolved_server_args(server_args: Any) -> Any:
"""Return SGLang's effective configuration for one initialized engine.
- SGLang #36255 exposes ``ServerArgs._resolved()``. Current SGLang keeps
- ``ServerArgs`` raw and exposes the same projection through
- ``resolved_view()``. Older supported releases and Dynamo's non-LLM argument
- stubs retain effective values on the object itself.
+ SGLang 0.5.18 and 0.5.19 keep ``ServerArgs`` raw and expose the effective
+ projection through ``resolved_view()``. The separately pinned XPU release
+ and Dynamo's non-LLM argument stubs retain effective values on the object
+ itself.
"""
- resolve = getattr(server_args, "_resolved", None)
- if callable(resolve):
- return resolve()
if sglang_resolved_view is not None:
return sglang_resolved_view(server_args)
return server_args
@@ -269,8 +352,12 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st
"ConfigArgumentMerger",
"ensure_sglang_tensor_image_size",
"filter_supported_async_generate_kwargs",
+ "get_encoder_preprocessor_modules",
+ "get_mm_encoder_class",
"get_sglang_model_config",
+ "mm_encode",
"override_server_args",
+ "publish_server_args",
"require_reasoning_kwargs",
"resolved_server_args",
"sglang_uses_mla_backend",
diff --git a/components/src/dynamo/sglang/init_multimodal.py b/components/src/dynamo/sglang/init_multimodal.py
index 7fe77cc00c81..d1daa4fe0e3a 100644
--- a/components/src/dynamo/sglang/init_multimodal.py
+++ b/components/src/dynamo/sglang/init_multimodal.py
@@ -18,6 +18,7 @@
WorkerType,
)
from dynamo.runtime import DistributedRuntime
+from dynamo.sglang._compat import publish_server_args
from dynamo.sglang.args import Config
from dynamo.sglang.health_check import (
SglangDisaggHealthCheckPayload,
@@ -60,6 +61,7 @@ async def init_multimodal_encode_worker(
cache_publisher = MultimodalEmbeddingCachePublisher()
await cache_publisher.create_endpoint(generate_endpoint)
+ publish_server_args(server_args, role="encoder")
handler = MultimodalEncodeWorkerHandler(
config,
pd_worker_client,
diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py
index 075ed2f40cb9..42209b9edd90 100644
--- a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py
+++ b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py
@@ -13,12 +13,11 @@
import torch
from blake3 import blake3
-# MMEncoder chain imports compiled CUDA ops; may fail in CPU-only environments.
+# Modality is safe to import during collection; MMEncoder itself is loaded
+# lazily by get_mm_encoder_class because it imports compiled CUDA operators.
try:
- from sglang.srt.disaggregation.encode_server import MMEncoder
from sglang.srt.managers.schedule_batch import Modality
except (ImportError, OSError):
- MMEncoder = None # type: ignore[assignment]
Modality = None # type: ignore[assignment]
from sglang.srt.parser.conversation import chat_templates
from transformers import AutoTokenizer
@@ -52,6 +51,11 @@
from dynamo.common.utils import nvtx_utils as _nvtx
from dynamo.common.utils.env import env_bool
from dynamo.llm import MultimodalEmbeddingCachePublisher
+from dynamo.sglang._compat import (
+ get_encoder_preprocessor_modules,
+ get_mm_encoder_class,
+ mm_encode,
+)
from dynamo.sglang.args import Config
from dynamo.sglang.protocol import (
MultiModalGroup,
@@ -144,16 +148,9 @@ def _install_load_video_passthrough() -> None:
Patch the name **as bound in each importing module**: they do
``from sglang.srt.utils import load_video``, so rebinding
``sglang.srt.utils.load_video`` alone leaves those call sites untouched.
- ``encode_server`` is the one this handler actually goes through -- its
- ``_flatten_and_load_videos`` calls its own binding -- and omitting it is why
- an earlier revision still raised ``ValueError: Unsupported video input type``
- end to end while every unit test passed.
-
- Verified against ``v0.5.16``: ``encode_server``, ``base_processor`` and
- ``utils`` are the ``srt`` modules that bind the name. Patch every one that
- imports successfully and log which, so a future SGLang bump that moves the
- call site shows up as a missing module rather than silently reverting to the
- URL path. Idempotent; a no-op when SGLang is unavailable.
+ The encoder preprocessor calls its own ``load_video`` binding, so patch that
+ module as well as the shared processor and utils bindings. Idempotent; a
+ no-op when SGLang is unavailable.
"""
if not SGLANG_VIDEO_DECODER_AVAILABLE:
return
@@ -171,17 +168,20 @@ def _load_video_passthrough(video_file, *args, **kwargs):
_load_video_passthrough._dynamo_nvdec_passthrough = True # type: ignore[attr-defined]
module.load_video = _load_video_passthrough
- patched: list[str] = []
+ encoder_modules = get_encoder_preprocessor_modules()
+ modules = list(encoder_modules)
for module_path in (
- # The encode worker's own call site -- the one that matters here.
- "sglang.srt.disaggregation.encode_server",
"sglang.srt.multimodal.processors.base_processor",
"sglang.srt.utils",
):
try:
- module = importlib.import_module(module_path)
+ modules.append(importlib.import_module(module_path))
except (ImportError, OSError):
continue
+
+ patched: list[str] = []
+ for module in modules:
+ module_path = module.__name__
orig = getattr(module, "load_video", None)
if orig is None:
continue
@@ -191,12 +191,14 @@ def _load_video_passthrough(video_file, *args, **kwargs):
_wrap(module, orig)
patched.append(module_path)
- if "sglang.srt.disaggregation.encode_server" not in patched:
+ encoder_module_names = {module.__name__ for module in encoder_modules}
+ if not encoder_module_names.intersection(patched):
# Without this one the decoder reaches load_video unpatched and the
# request fails with "Unsupported video input type" rather than falling
# back, so make the gap visible instead of waiting for a 400.
logger.warning(
- "load_video passthrough not installed on encode_server (patched: %s); "
+ "load_video passthrough not installed on encoder preprocessor "
+ "(patched: %s); "
"NVDEC video decoding will not work on this SGLang version",
patched or "none",
)
@@ -227,31 +229,27 @@ def _install_nvdec_video_metadata_shim() -> None:
so the synthesized values below never apply to it. The shim remains only for
genuine pre-decoded arrays.
"""
- try:
- from sglang.srt.disaggregation import encode_server as es
- except (ImportError, OSError):
- return
-
- orig = getattr(es, "preprocess_video", None)
- if orig is None or getattr(orig, "_dynamo_nvdec_shim", False):
- return
+ for module in get_encoder_preprocessor_modules():
+ orig = getattr(module, "preprocess_video", None)
+ if orig is None or getattr(orig, "_dynamo_nvdec_shim", False):
+ continue
- async def _preprocess_video_with_metadata(vr, *args, **kwargs):
- video, metadata = await orig(vr, *args, **kwargs)
- if metadata is None and isinstance(video, np.ndarray) and video.ndim >= 1:
- num_frames = int(video.shape[0])
- fps = _NVDEC_SHIM_FPS
- metadata = {
- "fps": fps,
- "duration": (num_frames / fps) if fps else 0.0,
- "total_num_frames": num_frames,
- "frames_indices": list(range(num_frames)),
- "video_backend": "nvdec",
- }
- return video, metadata
+ async def _preprocess_video_with_metadata(vr, *args, _orig=orig, **kwargs):
+ video, metadata = await _orig(vr, *args, **kwargs)
+ if metadata is None and isinstance(video, np.ndarray) and video.ndim >= 1:
+ num_frames = int(video.shape[0])
+ fps = _NVDEC_SHIM_FPS
+ metadata = {
+ "fps": fps,
+ "duration": (num_frames / fps) if fps else 0.0,
+ "total_num_frames": num_frames,
+ "frames_indices": list(range(num_frames)),
+ "video_backend": "nvdec",
+ }
+ return video, metadata
- _preprocess_video_with_metadata._dynamo_nvdec_shim = True # type: ignore[attr-defined]
- es.preprocess_video = _preprocess_video_with_metadata
+ _preprocess_video_with_metadata._dynamo_nvdec_shim = True # type: ignore[attr-defined]
+ module.preprocess_video = _preprocess_video_with_metadata # type: ignore[attr-defined]
class MultimodalEncodeWorkerHandler(BaseWorkerHandler[SglangMultimodalRequest, str]):
@@ -292,15 +290,17 @@ def __init__(
self.num_video_frames = max(1, VideoLoader.NUM_FRAMES_DEFAULT)
self._url_policy = UrlValidationPolicy.from_env()
- if MMEncoder is None:
+ try:
+ mm_encoder_class = get_mm_encoder_class()
+ except (ImportError, OSError) as exc:
raise RuntimeError(
"MMEncoder is not available. "
"Multimodal encode worker requires a CUDA environment."
- )
+ ) from exc
# torch.distributed requires a dist_init_method even for tp=1;
# port 0 lets the OS assign a free port.
- self.encoder = MMEncoder(
+ self.encoder = mm_encoder_class(
server_args=config.server_args,
dist_init_method="tcp://127.0.0.1:0",
rank=0,
@@ -786,7 +786,7 @@ async def _encode_with_cache(
encode_inputs = await self._build_encode_inputs(
uncached_inputs, modality_name
)
- grid_dim, new_embeddings, aux_data = await self.encoder._encode(
+ grid_dim, new_embeddings, aux_data = await self._encode_media(
encode_inputs, modality
)
# Verify SGLang output is on CPU as expected
@@ -860,6 +860,11 @@ async def _encode_with_cache(
full_embeddings = torch.cat(embedding_parts, dim=0)
return torch.tensor(all_grid_thw), full_embeddings, all_entries
+ async def _encode_media(
+ self, media_inputs: list[Any], modality: Any
+ ) -> tuple[Any, torch.Tensor, dict[str, Any]]:
+ return await mm_encode(self.encoder, media_inputs, modality)
+
def _extract_media_inputs(
self, request: Dict[str, Any]
) -> tuple[list[Any], list[str]]:
@@ -1120,7 +1125,7 @@ async def generate(
encode_inputs = await self._build_encode_inputs(
media_inputs, modality_name
)
- grid_dim, embeddings, aux_data = await self.encoder._encode(
+ grid_dim, embeddings, aux_data = await self._encode_media(
encode_inputs, modality_enum
)
diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py
index e99de7f35667..4d53e0a7d1bc 100644
--- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py
+++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py
@@ -4,6 +4,7 @@
"""Unit tests for SGLang multimodal embedding cache behavior."""
import asyncio
+import importlib
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -30,6 +31,7 @@
from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import (
Modality,
MultimodalEncodeWorkerHandler,
+ _install_load_video_passthrough,
)
pytestmark = [
@@ -975,8 +977,8 @@ async def _responses():
)
-def test_load_video_passthrough_patches_the_encode_server_binding() -> None:
- """The passthrough must reach the binding encode_server actually calls.
+def test_load_video_passthrough_patches_the_encoder_binding() -> None:
+ """The passthrough must reach the binding the encoder actually calls.
Regression: an earlier revision patched base_processor and sglang.srt.utils
but not encode_server, which does its own ``from sglang.srt.utils import
@@ -984,21 +986,22 @@ def test_load_video_passthrough_patches_the_encode_server_binding() -> None:
load_video was never reached -- while the deployed encode worker rejected
each video request with "Unsupported video input type" and returned 400.
"""
- encode_server = pytest.importorskip(
- "sglang.srt.disaggregation.encode_server",
- reason="SGLang required to verify the patch target",
- )
+ try:
+ encoder_preprocessor = importlib.import_module(
+ "sglang.srt.disaggregation.encoder.preprocessor"
+ )
+ except ImportError:
+ encoder_preprocessor = pytest.importorskip(
+ "sglang.srt.disaggregation.encode_server",
+ reason="SGLang required to verify the patch target",
+ )
from sglang.srt.utils.video_decoder import VideoDecoderWrapper
- from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import (
- _install_load_video_passthrough,
- )
-
_install_load_video_passthrough()
- patched = encode_server.load_video
+ patched = encoder_preprocessor.load_video
assert getattr(patched, "_dynamo_nvdec_passthrough", False), (
- "encode_server.load_video is unpatched; NVDEC decoders will be rejected "
- "by SGLang at request time"
+ "encoder preprocessor load_video is unpatched; NVDEC decoders will be "
+ "rejected by SGLang at request time"
)
# A pre-built decoder passes straight through instead of raising.
@@ -1015,4 +1018,4 @@ def test_load_video_passthrough_patches_the_encode_server_binding() -> None:
patched(object())
_install_load_video_passthrough() # idempotent
- assert encode_server.load_video is patched
+ assert encoder_preprocessor.load_video is patched
diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py
index 607d1a04a192..3631a96335f5 100644
--- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py
+++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py
@@ -22,8 +22,10 @@
StopConditions,
)
from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import (
+ _NVDEC_SHIM_FPS,
Modality,
MultimodalEncodeWorkerHandler,
+ _install_nvdec_video_metadata_shim,
)
from dynamo.sglang.request_handlers.multimodal.worker_handler import (
EmbeddingsProcessor,
@@ -106,6 +108,44 @@ def test_extract_media_inputs_supports_mixed_image_and_video():
assert video_urls == ["https://example.com/clip.mp4"]
+@pytest.mark.asyncio
+async def test_encode_media_uses_sglang_0_5_19_pipeline():
+ embeddings = torch.ones((4, 8))
+ preprocess_result = SimpleNamespace(grid_thw=[[1, 2, 2]])
+ encode_context = SimpleNamespace(
+ preprocess_result=preprocess_result,
+ aux_data={"example": "value"},
+ )
+ calls = []
+
+ class _Encoder:
+ async def _prepare_encode_context(
+ self, requests, modality, *, use_global_cache
+ ):
+ calls.append((requests, modality, use_global_cache))
+ return encode_context
+
+ async def _compute_embedding(self, context, *, keep_on_gpu):
+ assert context is encode_context
+ assert keep_on_gpu is False
+ return embeddings
+
+ handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler)
+ handler.encoder = _Encoder()
+
+ grid_thw, result, aux_data = await handler._encode_media(
+ ["https://example.com/image.png"], Modality.IMAGE
+ )
+
+ assert grid_thw == [[1, 2, 2]]
+ assert result is embeddings
+ assert aux_data == {"example": "value"}
+ requests, modality, use_global_cache = calls[0]
+ assert requests[0]["mm_items"] == ["https://example.com/image.png"]
+ assert modality == Modality.IMAGE
+ assert use_global_cache is False
+
+
@pytest.mark.multimodal
def test_extract_media_inputs_rejects_multimodal_cache_uuid():
handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler)
@@ -529,26 +569,30 @@ async def test_nvdec_video_metadata_shim_stamps_valid_metadata():
ndarray, and transformers >= 5.12 strict-rejects the resulting
``video_metadata=[None]``. ``_install_nvdec_video_metadata_shim`` wraps it so
the ndarray carries a valid dict instead (validated end-to-end on gpu-ts
- against the real ``MMEncoder._encode``: before FAIL -> after PASS).
+ against the real MMEncoder pipeline: before FAIL -> after PASS).
"""
- es = pytest.importorskip("sglang.srt.disaggregation.encode_server")
- from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import (
- _NVDEC_SHIM_FPS,
- _install_nvdec_video_metadata_shim,
- )
-
- saved = es.preprocess_video
+ try:
+ encoder_preprocessor = importlib.import_module(
+ "sglang.srt.disaggregation.encoder.preprocessor"
+ )
+ except ImportError:
+ encoder_preprocessor = pytest.importorskip(
+ "sglang.srt.disaggregation.encode_server"
+ )
+ saved = encoder_preprocessor.preprocess_video
try:
_install_nvdec_video_metadata_shim()
- assert getattr(es.preprocess_video, "_dynamo_nvdec_shim", False)
+ assert getattr(
+ encoder_preprocessor.preprocess_video, "_dynamo_nvdec_shim", False
+ )
# Idempotent: a second install must not double-wrap.
- wrapped = es.preprocess_video
+ wrapped = encoder_preprocessor.preprocess_video
_install_nvdec_video_metadata_shim()
- assert es.preprocess_video is wrapped
+ assert encoder_preprocessor.preprocess_video is wrapped
frames = np.zeros((5, 8, 8, 3), dtype=np.uint8)
- video, meta = await es.preprocess_video(frames)
+ video, meta = await encoder_preprocessor.preprocess_video(frames)
assert video is frames
assert meta == {
"fps": _NVDEC_SHIM_FPS,
@@ -559,10 +603,10 @@ async def test_nvdec_video_metadata_shim_stamps_valid_metadata():
}
# Non-ndarray inputs keep None metadata (the shim only touches pixels).
- _, meta_none = await es.preprocess_video("not-an-array")
+ _, meta_none = await encoder_preprocessor.preprocess_video("not-an-array")
assert meta_none is None
finally:
- es.preprocess_video = saved
+ encoder_preprocessor.preprocess_video = saved
# ---------------------------------------------------------------------------
diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py
index 1e8c18e84fbb..1d59854237cd 100644
--- a/components/src/dynamo/sglang/tests/test_sglang_unit.py
+++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py
@@ -25,6 +25,7 @@
filter_supported_async_generate_kwargs,
get_sglang_model_config,
override_server_args,
+ publish_server_args,
require_reasoning_kwargs,
resolved_server_args,
sglang_uses_mla_backend,
@@ -121,28 +122,6 @@ def declare(server_args, source, **fields):
assert not hasattr(server_args, "enable_memory_saver")
-def test_override_server_args_supports_sglang_0_5_17(monkeypatch):
- calls = []
-
- class ServerArgs:
- def override(self, source, **fields):
- calls.append((source, fields))
- for name, value in fields.items():
- object.__setattr__(self, name, value)
-
- monkeypatch.setattr(sglang_compat, "declare_late_resolution", None)
- server_args = ServerArgs()
-
- override_server_args(
- server_args,
- "dynamo.test",
- enable_memory_saver=True,
- )
-
- assert calls == [("dynamo.test", {"enable_memory_saver": True})]
- assert server_args.enable_memory_saver is True
-
-
def test_override_server_args_supports_legacy_xpu_pin(monkeypatch):
monkeypatch.setattr(sglang_compat, "declare_late_resolution", None)
server_args = SimpleNamespace(enable_memory_saver=False)
@@ -158,6 +137,20 @@ def test_override_server_args_supports_legacy_xpu_pin(monkeypatch):
assert server_args.load_format == "legacy-loader"
+def test_publish_server_args_uses_runtime_context(monkeypatch):
+ calls = []
+ server_args = SimpleNamespace()
+ monkeypatch.setattr(
+ sglang_compat,
+ "_sglang_publish",
+ lambda value, *, role: calls.append((value, role)),
+ )
+
+ publish_server_args(server_args, role="encoder")
+
+ assert calls == [(server_args, "encoder")]
+
+
def test_resolved_server_args_uses_declarative_view(monkeypatch):
raw_server_args = SimpleNamespace(page_size=None)
resolved_server_args_view = SimpleNamespace(page_size=64)
@@ -218,12 +211,15 @@ def test_compat_uses_legacy_sglang_mla_accessor(monkeypatch):
assert sglang_uses_mla_backend(server_args) is True
-def test_config_uses_resolved_server_args_after_runtime_init():
+def test_config_uses_resolved_server_args_after_runtime_init(monkeypatch):
raw_server_args = SimpleNamespace(page_size=None, disaggregation_mode="null")
resolved_server_args = SimpleNamespace(page_size=64, disaggregation_mode="null")
- raw_server_args._resolved = lambda: resolved_server_args
+ monkeypatch.setattr(
+ sglang_compat,
+ "sglang_resolved_view",
+ lambda server_args: resolved_server_args,
+ )
config = sglang_args.Config(raw_server_args, SimpleNamespace())
-
runtime_server_args = config.use_resolved_server_args(raw_server_args)
assert raw_server_args.page_size is None
diff --git a/container/compliance/base_sboms/manifest.json b/container/compliance/base_sboms/manifest.json
index 5bc5a7f9402a..5b1a43a33906 100644
--- a/container/compliance/base_sboms/manifest.json
+++ b/container/compliance/base_sboms/manifest.json
@@ -110,10 +110,10 @@
"baseline_image": "nvidia/cuda",
"baseline_sbom": "cuda@0230b7f2-amd64.cdx.json",
"baseline_tag": "13.0.3-cudnn-devel-ubuntu24.04",
- "capture_note": "lmsysorg/sglang is built on this nvidia/cuda cudnn-devel base; the vendor image layout is not layer-prefix preserving, so layer-prefix validation was skipped. All from-image delta components passed policy validation at capture.",
- "from_digest": "sha256:d89bf566cec87e4beed8ee4068b409552eef1f83d3d9c9efb94a6e98753a70cd",
+ "capture_note": "lmsysorg/sglang is built on this nvidia/cuda cudnn-devel base; the vendor image layout is not layer-prefix preserving, so layer-prefix validation was skipped. Registry digest and CUDA 13.0.3 metadata were verified locally; full from-image delta policy validation runs in CI.",
+ "from_digest": "sha256:710bc11443a7b1807d69803386468101bcfced35f86bf8fe92a8209e05a2f052",
"from_image": "lmsysorg/sglang",
- "from_tag": "v0.5.18-cu130-runtime",
+ "from_tag": "v0.5.19-cu130-runtime",
"layer_prefix_check_skipped": true,
"platform": "linux/amd64"
},
@@ -122,10 +122,10 @@
"baseline_image": "nvidia/cuda",
"baseline_sbom": "cuda@0230b7f2-arm64.cdx.json",
"baseline_tag": "13.0.3-cudnn-devel-ubuntu24.04",
- "capture_note": "lmsysorg/sglang is built on this nvidia/cuda cudnn-devel base; the vendor image layout is not layer-prefix preserving, so layer-prefix validation was skipped. All from-image delta components passed policy validation at capture.",
- "from_digest": "sha256:d89bf566cec87e4beed8ee4068b409552eef1f83d3d9c9efb94a6e98753a70cd",
+ "capture_note": "lmsysorg/sglang is built on this nvidia/cuda cudnn-devel base; the vendor image layout is not layer-prefix preserving, so layer-prefix validation was skipped. Registry digest and CUDA 13.0.3 metadata were verified locally; full from-image delta policy validation runs in CI.",
+ "from_digest": "sha256:710bc11443a7b1807d69803386468101bcfced35f86bf8fe92a8209e05a2f052",
"from_image": "lmsysorg/sglang",
- "from_tag": "v0.5.18-cu130-runtime",
+ "from_tag": "v0.5.19-cu130-runtime",
"layer_prefix_check_skipped": true,
"platform": "linux/arm64"
},
@@ -162,6 +162,6 @@
"platform": "platform pinned for layer-prefix verification, e.g. linux/amd64"
},
"format": "CycloneDX 1.6 JSON, slim filter (drop properties/hashes/dependencies; keep evidence). Hard cap 5 MB per file.",
- "generated_at": "2026-09-01T00:35:32+00:00",
+ "generated_at": "2026-09-08T22:54:27+00:00",
"schema_version": 2
}
diff --git a/container/context.yaml b/container/context.yaml
index 685930a4c3e4..5b2c03db3f72 100644
--- a/container/context.yaml
+++ b/container/context.yaml
@@ -123,7 +123,7 @@ sglang:
base_image: nvcr.io/nvidia/cuda-dl-base
runtime_image: lmsysorg/sglang
base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04
- runtime_image_tag: v0.5.18-cu130-runtime
+ runtime_image_tag: v0.5.19-cu130-runtime
# Baseline is the TRUE base of the third-party lmsysorg/sglang image,
# nvidia/cuda:13.0.3-cudnn-devel-ubuntu24.04 (Docker Hub; CUDA_VERSION=13.0.3,
# ubuntu24.04), NOT a self-baseline of lmsysorg/sglang.
diff --git a/container/templates/dev.Dockerfile b/container/templates/dev.Dockerfile
index 6407b25e151e..7ab055b0e726 100644
--- a/container/templates/dev.Dockerfile
+++ b/container/templates/dev.Dockerfile
@@ -355,11 +355,13 @@ RUN cp /tmp/uv-binary ${VIRTUAL_ENV}/bin/uv && \
chmod +x ${VIRTUAL_ENV}/bin/uv && \
pip install maturin[patchelf]
{% else %}
-# SGLang CUDA: Create venv with --system-site-packages to inherit runtime packages
+# SGLang CUDA: Create a writable Dynamo venv and seed it from the upstream
+# SGLang venv. The 0.5.19 runtime moved its packages from the system Python's
+# dist-packages directory to /opt/sglang.
COPY --from=ghcr.io/astral-sh/uv:{{ context.dynamo.uv_version }} /uv /tmp/uv-binary
RUN mkdir -p /opt/dynamo/venv && \
python3 -m venv --system-site-packages /opt/dynamo/venv && \
- cp -r /usr/local/lib/python${PYTHON_VERSION}/dist-packages/* \
+ cp -r /opt/sglang/lib/python${PYTHON_VERSION}/site-packages/. \
/opt/dynamo/venv/lib/python${PYTHON_VERSION}/site-packages/ && \
chmod -R g+w /opt/dynamo/venv/lib/python${PYTHON_VERSION}/site-packages/ && \
cp /tmp/uv-binary /opt/dynamo/venv/bin/uv && \
diff --git a/docs/fern/components/releases.data.ts b/docs/fern/components/releases.data.ts
index c53892f35381..7afa12ea2c3f 100644
--- a/docs/fern/components/releases.data.ts
+++ b/docs/fern/components/releases.data.ts
@@ -71,7 +71,7 @@ export const CURRENT_TAG = "1.4.2";
export const CURRENT_WHEEL = "1.4.2";
export const MAIN_TOT: BackendPins = {
- sglang: "0.5.18",
+ sglang: "0.5.19",
trtllm: "1.3.0rc25",
vllm: "0.28.0",
nixlSglang: "1.4.0",
@@ -1210,7 +1210,7 @@ export const FEATURE_INTERACTIONS: BackendInteractions[] = [
// KV Block Manager
[{ status: "wip" }, { status: "wip" }, { status: "wip" }, { status: "na" }],
// Multimodal
- [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.18, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }],
+ [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.19, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }],
// Request Migration
[{ status: "yes" }, { status: "yes" }, { status: "yes" }, { status: "wip" }, { status: "yes" }, { status: "na" }],
// Request Cancellation
diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md
index 30e7b870d175..8c4cb9819b03 100644
--- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md
+++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md
@@ -122,7 +122,7 @@ The launcher configures KV events on each worker and sets `--router-mode kv` wit
The Dynamo SGLang image includes both routing prerequisites:
- Dynamo is built with the `mm-routing` Rust feature.
-- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.18.
+- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.19.
Custom installations on SGLang 0.5.12 or earlier need the `mm_hashes` change
from [sgl-project/sglang#25300](https://github.com/sgl-project/sglang/pull/25300).
@@ -551,7 +551,7 @@ Controls how many threads the encoder uses to fetch and load images concurrently
export SGLANG_ENCODER_MM_LOAD_WORKERS=16
```
-Only applies to the EPD encode worker (which uses [SGLang's MMEncoder](https://github.com/sgl-project/sglang/blob/v0.5.18/python/sglang/srt/disaggregation/encode_server.py) internally).
+Only applies to the EPD encode worker (which uses [SGLang's MMEncoder](https://github.com/sgl-project/sglang/blob/v0.5.19/python/sglang/srt/disaggregation/encoder/server.py) internally).
## Profiling
diff --git a/docs/fern/pages/reference/general/compatibility.mdx b/docs/fern/pages/reference/general/compatibility.mdx
index 9b3755799e10..92258819a1f6 100644
--- a/docs/fern/pages/reference/general/compatibility.mdx
+++ b/docs/fern/pages/reference/general/compatibility.mdx
@@ -172,7 +172,7 @@ Current stable release: v1.4.2 (container tag `1.4.2`, wheel version `1.4.2`).
| Dynamo | Type | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX |
| --- | --- | --- | --- | --- | --- | --- |
-| main (ToT) | development head | 0.5.18 | 1.3.0rc25 | 0.28.0 | 1.4.0 / 1.3.1 / 1.3.2 | - |
+| main (ToT) | development head | 0.5.19 | 1.3.0rc25 | 0.28.0 | 1.4.0 / 1.3.1 / 1.3.2 | - |
| v1.4.2 | patch | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x |
| v1.4.1 | patch | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x |
| v1.4.0 | stable | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x |
@@ -330,10 +330,10 @@ Each cell states whether the row feature works together with the column feature.
| Feature | Disaggregated Serving | KV-Aware Routing | SLA-Based Planner | KV Block Manager | Multimodal | Request Migration | Request Cancellation | LoRA | Tool Calling | Speculative Decoding |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
| Disaggregated Serving | n/a | Yes | Yes | Experimental | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental — Prefill/decode lifecycle registration has unit coverage, but no SGLang disaggregated LoRA end-to-end test. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental — Code hooks exist, but examples and documentation are not yet available. |
-| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.18, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | Experimental — Aggregated LoRA inference is validated without the KV router; the combined path remains experimental. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental |
+| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.19, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | Experimental — Aggregated LoRA inference is validated without the KV router; the combined path remains experimental. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental |
| SLA-Based Planner | Yes | Yes | n/a | Experimental | n/a | Yes | Yes | n/a | Yes | n/a |
| KV Block Manager | Experimental | Experimental | Experimental | n/a | Experimental | Experimental | Experimental | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental | Experimental |
-| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.18, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | n/a | Yes | n/a |
+| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.19, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | n/a | Yes | n/a |
| Request Migration | Yes | Yes | Yes | Experimental | Yes | n/a | Yes | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental |
| Request Cancellation | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Yes | Experimental | Experimental | Yes | n/a | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a |
| LoRA | Experimental — Prefill/decode lifecycle registration has unit coverage, but no SGLang disaggregated LoRA end-to-end test. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental — Aggregated LoRA inference is validated without the KV router; the combined path remains experimental. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | Experimental — Tool calling with SGLang LoRA is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental — Speculative decoding with SGLang LoRA is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) |
diff --git a/docs/fern/pages/reference/general/releases-machine-readable.mdx b/docs/fern/pages/reference/general/releases-machine-readable.mdx
index f39ec24cc0f7..d22f8e9d450e 100644
--- a/docs/fern/pages/reference/general/releases-machine-readable.mdx
+++ b/docs/fern/pages/reference/general/releases-machine-readable.mdx
@@ -15,7 +15,7 @@ Current stable release: v1.4.2 (Aug 28, 2026; container tag `1.4.2`, wheel versi
| Version | Kind | Date | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | Notes | Delta |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
-| main (ToT) | development head | - | 0.5.18 | 1.3.0rc25 | 0.28.0 | 1.4.0 / 1.3.1 / 1.3.2 | - | - | - |
+| main (ToT) | development head | - | 0.5.19 | 1.3.0rc25 | 0.28.0 | 1.4.0 / 1.3.1 / 1.3.2 | - | - | - |
| v1.4.2 | patch | Aug 28, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0#v142) | Patch release and the first Dynamo Enterprise release: a curated set of release artifacts publishes under the -enterprise suffix on NGC, eligible for NVIDIA Enterprise Support, with no functional or binary differences from the open-source artifacts. Fixes NIXL loader-path resolution in the Frontend and SGLang Runtime images, removes the unused Nsight EFA metrics plugin, and tightens dependency pins (pillow v12.3.0 floor, plotext below v6, EFA Installer v1.50). Backend pins are unchanged from v1.4.0. |
| v1.4.1 | patch | Aug 21, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0#v141) | Patch release. Adds the classify and pooling endpoints, forwards logprob_token_ids through the OpenAI frontend, reconciles request-path overload marks in the Router, and fixes NIXL writable buffers for vLLM. All three Go modules move to Go 1.26.6 with aligned x/net and grpc. Backend pins are unchanged from v1.4.0. |
| v1.4.0 | stable | Aug 14, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0) | Audit subsystem migrated into request trace (DYN_AUDIT_* honored as legacy aliases); HTTP header capture in trace records is an explicit fail-closed allowlist; deprecated multimodal worker flags and vLLM worker-role flags removed; runtime images no longer bundle software video decoders (H.264/H.265 decodes via NVDEC); UCX 1.21.x. |
diff --git a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md
index 8a40033a771e..98a6e605e024 100644
--- a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md
+++ b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md
@@ -113,5 +113,5 @@ URI string, while frontend decoding hashes the decoded bytes.
|---------|--------------|--------|-------|
| [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported families include Qwen2-VL, Qwen2.5-VL, Qwen3-VL, LLaVA 1.5, LLaVA-NeXT, Llama 4, Kimi K2.5/K2.6, Qwen3.5, and Qwen3.6. The rest use text-prefix-only routing. |
| [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Python chat processor | Yes | Uses vLLM’s own multimodal processor — supports any VLM that vLLM supports. |
-| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.18. Older custom installations need the upstream patch. |
+| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.19. Older custom installations need the upstream patch. |
| [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported model scope is the Qwen2-VL family (Qwen2-VL / Qwen2.5-VL / Qwen3-VL) and Kimi (Kimi-K2.5 / Kimi-K2.6). Other multimodal models fall back to text-prefix routing. |
diff --git a/lib/sidecar/Dockerfile b/lib/sidecar/Dockerfile
index 7443404aa2a7..231e7b08b5f4 100644
--- a/lib/sidecar/Dockerfile
+++ b/lib/sidecar/Dockerfile
@@ -73,6 +73,7 @@ COPY .cargo/ .cargo/
COPY lib/ lib/
COPY examples/router/custom-policy-example/ examples/router/custom-policy-example/
COPY deploy/inference-gateway/ext-proc/ deploy/inference-gateway/ext-proc/
+COPY deploy/inference-gateway/sidecar/ deploy/inference-gateway/sidecar/
# ---- vLLM builder -----------------------------------------------------------
FROM builder-base AS vllm-builder
diff --git a/lib/sidecar/sglang/README.md b/lib/sidecar/sglang/README.md
index faf4a528a128..cba07fd3c117 100644
--- a/lib/sidecar/sglang/README.md
+++ b/lib/sidecar/sglang/README.md
@@ -77,7 +77,7 @@ command.
> The engine image must be a stock SGLang **v0.5.16+** build: the native gRPC
> server (`--grpc-port`) landed there. The KV-routing examples require
> **v0.5.18+** because the sidecar discovers their structured KV-event
-> descriptor through `GetServerInfo`. They use `lmsysorg/sglang:v0.5.18`.
+> descriptor through `GetServerInfo`. They use `lmsysorg/sglang:v0.5.19`.
### Prerequisites
diff --git a/lib/sidecar/sglang/deploy/agg_kv_router.yaml b/lib/sidecar/sglang/deploy/agg_kv_router.yaml
index 85f140f6d4c9..6c3d82d97405 100644
--- a/lib/sidecar/sglang/deploy/agg_kv_router.yaml
+++ b/lib/sidecar/sglang/deploy/agg_kv_router.yaml
@@ -42,7 +42,7 @@ spec:
# separate pod network namespace.
initContainers:
- name: sglang-engine
- image: lmsysorg/sglang:v0.5.18
+ image: lmsysorg/sglang:v0.5.19
restartPolicy: Always
startupProbe:
exec:
diff --git a/lib/sidecar/sglang/deploy/disagg_kv_router.yaml b/lib/sidecar/sglang/deploy/disagg_kv_router.yaml
index dd4b6a5756d8..a30165d75848 100644
--- a/lib/sidecar/sglang/deploy/disagg_kv_router.yaml
+++ b/lib/sidecar/sglang/deploy/disagg_kv_router.yaml
@@ -50,7 +50,7 @@ spec:
sizeLimit: 10Gi
initContainers:
- name: sglang-engine
- image: lmsysorg/sglang:v0.5.18
+ image: lmsysorg/sglang:v0.5.19
restartPolicy: Always
securityContext:
capabilities:
@@ -144,7 +144,7 @@ spec:
sizeLimit: 10Gi
initContainers:
- name: sglang-engine
- image: lmsysorg/sglang:v0.5.18
+ image: lmsysorg/sglang:v0.5.19
restartPolicy: Always
securityContext:
capabilities:
diff --git a/pyproject.toml b/pyproject.toml
index ab862db2cea5..8dd83d0e570e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -74,7 +74,7 @@ vllm = [
sglang = [
"uvloop",
- "sglang[diffusion]==0.5.18",
+ "sglang[diffusion]==0.5.19",
# sglang[diffusion] dropped accelerate in 0.5.12; diffusers still needs it.
"accelerate>=0.17.0",
"blake3>=1.0.0,<2.0.0",
diff --git a/tests/report_pytest_markers.py b/tests/report_pytest_markers.py
index 0efdcbe2194c..fcdad9c0692f 100755
--- a/tests/report_pytest_markers.py
+++ b/tests/report_pytest_markers.py
@@ -180,6 +180,7 @@
"sglang.srt.utils",
"sglang.srt.utils.hf_transformers_utils",
"sglang.srt.utils.network",
+ "sglang.srt.utils.server_args_config_parser",
"sglang.srt.utils.video_decoder",
"sglang.srt.disaggregation",
"sglang.srt.disaggregation.kv_events",
diff --git a/tests/serve/common.py b/tests/serve/common.py
index 602f324801e1..e6e101fe055d 100644
--- a/tests/serve/common.py
+++ b/tests/serve/common.py
@@ -6,8 +6,10 @@
import dataclasses
import logging
import os
+import shutil
import subprocess
import sys
+import tempfile
import time
from collections.abc import Callable, Mapping
from contextlib import contextmanager
@@ -307,32 +309,63 @@ def managed_serve_deployment(
# is retained without the shipped image carrying it. No-op when the key is unset.
TEST_ONLY_PIP_ENV_KEY = "DYN_TEST_ONLY_PIP_INSTALL"
-# Session-level guard so the same package set is installed at most once even
+# Session-level cache so the same package set is installed at most once even
# though every parametrized deployment (and each retry) calls the installer.
-_test_only_pip_done: set[str] = set()
+_test_only_pip_targets: dict[str, str] = {}
-def _install_test_only_packages(config: EngineConfig) -> None:
+def _install_test_only_packages(
+ config: EngineConfig, extra_env: Optional[Dict[str, str]] = None
+) -> dict[str, str]:
"""Install any test-only pip packages a config requested via its env.
- Runs inside the same runtime container/interpreter the server subprocess
- inherits, so the worker can import the freshly installed module.
+ Install into a process-isolated temporary directory rather than the runtime
+ interpreter's site-packages. CI may run the image as an arbitrary uid, and
+ some framework venvs are intentionally read-only. The returned environment
+ exposes the directory only to subprocesses launched for this deployment.
"""
+ launch_env = dict(extra_env or {})
spec = config.env.get(TEST_ONLY_PIP_ENV_KEY, "").strip()
- if not spec or spec in _test_only_pip_done:
- return
- packages = spec.split()
- logging.getLogger(__name__).info(
- "Installing test-only package(s) into runtime container: %s",
- " ".join(packages),
+ if not spec:
+ return launch_env
+
+ target = _test_only_pip_targets.get(spec)
+ if target is None:
+ packages = spec.split()
+ target = tempfile.mkdtemp(prefix="dynamo-test-pip-")
+ logging.getLogger(__name__).info(
+ "Installing test-only package(s) into %s: %s",
+ target,
+ " ".join(packages),
+ )
+ try:
+ subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "pip",
+ "install",
+ "--target",
+ target,
+ "--no-deps",
+ *packages,
+ ],
+ check=True,
+ )
+ except Exception:
+ shutil.rmtree(target, ignore_errors=True)
+ raise
+ _test_only_pip_targets[spec] = target
+
+ inherited_pythonpath = launch_env.get(
+ "PYTHONPATH", config.env.get("PYTHONPATH", os.environ.get("PYTHONPATH", ""))
)
- # --break-system-packages: runtime images use an externally-managed system
- # python (PEP 668); this is the ephemeral test container, not a shipped image.
- subprocess.run(
- [sys.executable, "-m", "pip", "install", "--break-system-packages", *packages],
- check=True,
+ launch_env["PYTHONPATH"] = (
+ target
+ if not inherited_pythonpath
+ else os.pathsep.join((target, inherited_pythonpath))
)
- _test_only_pip_done.add(spec)
+ return launch_env
def run_serve_deployment(
@@ -363,7 +396,7 @@ def run_serve_deployment(
# Install any decoder a codec-stripped image needs for this test, before the
# server launches, so the worker can import it. No-op unless the config opts in.
- _install_test_only_packages(config)
+ extra_env = _install_test_only_packages(config, extra_env)
prep = _prepare_deployment(config, request, ports=ports, extra_env=extra_env)
config = prep.config
diff --git a/tests/serve/test_common_helpers.py b/tests/serve/test_common_helpers.py
new file mode 100644
index 000000000000..7d5ed7ec9ac0
--- /dev/null
+++ b/tests/serve/test_common_helpers.py
@@ -0,0 +1,101 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+from pathlib import Path
+
+import pytest
+
+from tests.serve import common
+from tests.utils.engine_process import EngineConfig
+
+pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0]
+
+
+def _config(*, spec: str = "") -> EngineConfig:
+ env = {common.TEST_ONLY_PIP_ENV_KEY: spec} if spec else {}
+ return EngineConfig(
+ name="test",
+ directory=".",
+ marks=[],
+ request_payloads=[],
+ model="test",
+ command=["true"],
+ env=env,
+ )
+
+
+@pytest.fixture(autouse=True)
+def clear_test_only_pip_targets():
+ common._test_only_pip_targets.clear()
+ yield
+ common._test_only_pip_targets.clear()
+
+
+def test_install_test_only_packages_uses_isolated_target(monkeypatch, tmp_path):
+ target = tmp_path / "packages"
+ calls = []
+ monkeypatch.setattr(common.tempfile, "mkdtemp", lambda **_kwargs: str(target))
+ monkeypatch.setattr(
+ common.subprocess,
+ "run",
+ lambda cmd, **kwargs: calls.append((cmd, kwargs)),
+ )
+
+ env = common._install_test_only_packages(
+ _config(spec="decord2>=3.4.0,<4"), {"PYTHONPATH": "/existing"}
+ )
+
+ assert calls == [
+ (
+ [
+ common.sys.executable,
+ "-m",
+ "pip",
+ "install",
+ "--target",
+ str(target),
+ "--no-deps",
+ "decord2>=3.4.0,<4",
+ ],
+ {"check": True},
+ )
+ ]
+ assert env["PYTHONPATH"] == f"{target}{common.os.pathsep}/existing"
+
+
+def test_install_test_only_packages_reuses_successful_install(monkeypatch, tmp_path):
+ target = tmp_path / "packages"
+ calls = []
+ monkeypatch.setattr(common.tempfile, "mkdtemp", lambda **_kwargs: str(target))
+ monkeypatch.setattr(
+ common.subprocess,
+ "run",
+ lambda cmd, **kwargs: calls.append((cmd, kwargs)),
+ )
+ config = _config(spec="decord2>=3.4.0,<4")
+
+ first = common._install_test_only_packages(config)
+ second = common._install_test_only_packages(config)
+
+ assert len(calls) == 1
+ assert first["PYTHONPATH"] == str(target)
+ assert second["PYTHONPATH"] == str(target)
+
+
+def test_install_test_only_packages_removes_failed_target(monkeypatch, tmp_path):
+ target = tmp_path / "packages"
+ target.mkdir()
+ marker = target / "partial-wheel"
+ marker.touch()
+ monkeypatch.setattr(common.tempfile, "mkdtemp", lambda **_kwargs: str(target))
+
+ def fail_install(*_args, **_kwargs):
+ raise RuntimeError("pip failed")
+
+ monkeypatch.setattr(common.subprocess, "run", fail_install)
+
+ with pytest.raises(RuntimeError, match="pip failed"):
+ common._install_test_only_packages(_config(spec="decord2>=3.4.0,<4"))
+
+ assert not Path(target).exists()
+ assert not common._test_only_pip_targets