From fb2fd06bb482ab50a5192052a70bdc85dfde16cd Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 1 Sep 2026 17:37:18 -0700 Subject: [PATCH 01/12] build(sglang): prepare version 0.5.19 bump Signed-off-by: jain-ria --- docs/fern/components/releases.data.ts | 4 ++-- .../modular-components/backends/sglang/multimodal.md | 4 ++-- docs/fern/pages/reference/general/compatibility.mdx | 6 +++--- .../pages/reference/general/releases-machine-readable.mdx | 2 +- .../use-cases/multimodal-serving/multimodal-kv-routing.md | 2 +- pyproject.toml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/fern/components/releases.data.ts b/docs/fern/components/releases.data.ts index c53892f35381..7afa12ea2c3f 100644 --- a/docs/fern/components/releases.data.ts +++ b/docs/fern/components/releases.data.ts @@ -71,7 +71,7 @@ export const CURRENT_TAG = "1.4.2"; export const CURRENT_WHEEL = "1.4.2"; export const MAIN_TOT: BackendPins = { - sglang: "0.5.18", + sglang: "0.5.19", trtllm: "1.3.0rc25", vllm: "0.28.0", nixlSglang: "1.4.0", @@ -1210,7 +1210,7 @@ export const FEATURE_INTERACTIONS: BackendInteractions[] = [ // KV Block Manager [{ status: "wip" }, { status: "wip" }, { status: "wip" }, { status: "na" }], // Multimodal - [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.18, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }], + [{ status: "yes", label: "Supported serving patterns", note: "Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported.", source: "/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal" }, { status: "yes", label: "Image-aware routing on Dynamo's SGLang image", note: "Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.19, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing.", source: "/dynamo/dev/multimodal/multimodal-kv-routing" }, { status: "na" }, { status: "wip" }, { status: "na" }], // Request Migration [{ status: "yes" }, { status: "yes" }, { status: "yes" }, { status: "wip" }, { status: "yes" }, { status: "na" }], // Request Cancellation diff --git a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md index 30e7b870d175..8c4cb9819b03 100644 --- a/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md +++ b/docs/fern/pages/developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md @@ -122,7 +122,7 @@ The launcher configures KV events on each worker and sets `--router-mode kv` wit The Dynamo SGLang image includes both routing prerequisites: - Dynamo is built with the `mm-routing` Rust feature. -- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.18. +- SGLang 0.5.13 or later includes `GenerateReqInput.mm_hashes` support. Dynamo currently pins 0.5.19. Custom installations on SGLang 0.5.12 or earlier need the `mm_hashes` change from [sgl-project/sglang#25300](https://github.com/sgl-project/sglang/pull/25300). @@ -551,7 +551,7 @@ Controls how many threads the encoder uses to fetch and load images concurrently export SGLANG_ENCODER_MM_LOAD_WORKERS=16 ``` -Only applies to the EPD encode worker (which uses [SGLang's MMEncoder](https://github.com/sgl-project/sglang/blob/v0.5.18/python/sglang/srt/disaggregation/encode_server.py) internally). +Only applies to the EPD encode worker (which uses [SGLang's MMEncoder](https://github.com/sgl-project/sglang/blob/v0.5.19/python/sglang/srt/disaggregation/encoder/server.py) internally). ## Profiling diff --git a/docs/fern/pages/reference/general/compatibility.mdx b/docs/fern/pages/reference/general/compatibility.mdx index 55eb22e095ef..569d1aed4971 100644 --- a/docs/fern/pages/reference/general/compatibility.mdx +++ b/docs/fern/pages/reference/general/compatibility.mdx @@ -172,7 +172,7 @@ Current stable release: v1.4.2 (container tag `1.4.2`, wheel version `1.4.2`). | Dynamo | Type | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | | --- | --- | --- | --- | --- | --- | --- | -| main (ToT) | development head | 0.5.18 | 1.3.0rc25 | 0.28.0 | 1.4.0 / 1.3.1 / 1.3.2 | - | +| main (ToT) | development head | 0.5.19 | 1.3.0rc25 | 0.28.0 | 1.4.0 / 1.3.1 / 1.3.2 | - | | v1.4.2 | patch | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | | v1.4.1 | patch | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | | v1.4.0 | stable | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | @@ -330,10 +330,10 @@ Each cell states whether the row feature works together with the column feature. | Feature | Disaggregated Serving | KV-Aware Routing | SLA-Based Planner | KV Block Manager | Multimodal | Request Migration | Request Cancellation | LoRA | Tool Calling | Speculative Decoding | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | | Disaggregated Serving | n/a | Yes | Yes | Experimental | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental — Prefill/decode lifecycle registration has unit coverage, but no SGLang disaggregated LoRA end-to-end test. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental — Code hooks exist, but examples and documentation are not yet available. | -| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.18, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | Experimental — Aggregated LoRA inference is validated without the KV router; the combined path remains experimental. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | +| KV-Aware Routing | Yes | n/a | Yes | Experimental | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.19, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | Yes | Yes | Experimental — Aggregated LoRA inference is validated without the KV router; the combined path remains experimental. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | | SLA-Based Planner | Yes | Yes | n/a | Experimental | n/a | Yes | Yes | n/a | Yes | n/a | | KV Block Manager | Experimental | Experimental | Experimental | n/a | Experimental | Experimental | Experimental | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental | Experimental | -| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.18, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | n/a | Yes | n/a | +| Multimodal | Yes — Supports aggregated EPD, E/PD, and E/P/D patterns. Traditional disaggregated EP/D is not supported. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/sglang-multimodal) | Yes — Hash forwarding is upstream in SGLang 0.5.13+ and Dynamo pins 0.5.19, so the shipped image routes on image overlap. A custom build without that patch still serves the request but degrades to text-prefix routing. (https://docs.nvidia.com/dynamo/dev/multimodal/multimodal-kv-routing) | n/a | Experimental | n/a | Yes | Experimental | n/a | Yes | n/a | | Request Migration | Yes | Yes | Yes | Experimental | Yes | n/a | Yes | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Experimental | | Request Cancellation | Experimental — Cancellation during remote prefill is not supported in disaggregated mode. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | Yes | Experimental | Experimental | Yes | n/a | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Yes | n/a | | LoRA | Experimental — Prefill/decode lifecycle registration has unit coverage, but no SGLang disaggregated LoRA end-to-end test. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental — Aggregated LoRA inference is validated without the KV router; the combined path remains experimental. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental — This LoRA feature pairing is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | n/a | Experimental — Tool calling with SGLang LoRA is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | Experimental — Speculative decoding with SGLang LoRA is not end-to-end validated. (https://docs.nvidia.com/dynamo/dev/knowledge-base/modular-components/backends/sg-lang/overview) | diff --git a/docs/fern/pages/reference/general/releases-machine-readable.mdx b/docs/fern/pages/reference/general/releases-machine-readable.mdx index f39ec24cc0f7..d22f8e9d450e 100644 --- a/docs/fern/pages/reference/general/releases-machine-readable.mdx +++ b/docs/fern/pages/reference/general/releases-machine-readable.mdx @@ -15,7 +15,7 @@ Current stable release: v1.4.2 (Aug 28, 2026; container tag `1.4.2`, wheel versi | Version | Kind | Date | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | Notes | Delta | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| main (ToT) | development head | - | 0.5.18 | 1.3.0rc25 | 0.28.0 | 1.4.0 / 1.3.1 / 1.3.2 | - | - | - | +| main (ToT) | development head | - | 0.5.19 | 1.3.0rc25 | 0.28.0 | 1.4.0 / 1.3.1 / 1.3.2 | - | - | - | | v1.4.2 | patch | Aug 28, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0#v142) | Patch release and the first Dynamo Enterprise release: a curated set of release artifacts publishes under the -enterprise suffix on NGC, eligible for NVIDIA Enterprise Support, with no functional or binary differences from the open-source artifacts. Fixes NIXL loader-path resolution in the Frontend and SGLang Runtime images, removes the unused Nsight EFA metrics plugin, and tightens dependency pins (pillow v12.3.0 floor, plotext below v6, EFA Installer v1.50). Backend pins are unchanged from v1.4.0. | | v1.4.1 | patch | Aug 21, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0#v141) | Patch release. Adds the classify and pooling endpoints, forwards logprob_token_ids through the OpenAI frontend, reconciles request-path overload marks in the Router, and fixes NIXL writable buffers for vLLM. All three Go modules move to Go 1.26.6 with aligned x/net and grpc. Backend pins are unchanged from v1.4.0. | | v1.4.0 | stable | Aug 14, 2026 | 0.5.16 | 1.3.0rc22 | 0.26.0 | 1.3.0 / 1.3.1 / 1.3.2 | 1.21.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-4-0) | Audit subsystem migrated into request trace (DYN_AUDIT_* honored as legacy aliases); HTTP header capture in trace records is an explicit fail-closed allowlist; deprecated multimodal worker flags and vLLM worker-role flags removed; runtime images no longer bundle software video decoders (H.264/H.265 decodes via NVDEC); UCX 1.21.x. | diff --git a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md index 8a40033a771e..98a6e605e024 100644 --- a/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md +++ b/docs/fern/pages/use-cases/multimodal-serving/multimodal-kv-routing.md @@ -113,5 +113,5 @@ URI string, while frontend decoding hashes the decoded bytes. |---------|--------------|--------|-------| | [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported families include Qwen2-VL, Qwen2.5-VL, Qwen3-VL, LLaVA 1.5, LLaVA-NeXT, Llama 4, Kimi K2.5/K2.6, Qwen3.5, and Qwen3.6. The rest use text-prefix-only routing. | | [vLLM](../../developer-guide/knowledge-base/modular-components/backends/vllm/multimodal.md#multimodal-kv-routing) | Python chat processor | Yes | Uses vLLM’s own multimodal processor — supports any VLM that vLLM supports. | -| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.18. Older custom installations need the upstream patch. | +| [SGLang](../../developer-guide/knowledge-base/modular-components/backends/sglang/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Hash forwarding is upstream in SGLang 0.5.13+; Dynamo pins 0.5.19. Older custom installations need the upstream patch. | | [TensorRT-LLM](../../developer-guide/knowledge-base/modular-components/backends/tensorrt-llm/multimodal.md#multimodal-kv-routing) | Rust frontend (default) | Yes | Supported model scope is the Qwen2-VL family (Qwen2-VL / Qwen2.5-VL / Qwen3-VL) and Kimi (Kimi-K2.5 / Kimi-K2.6). Other multimodal models fall back to text-prefix routing. | diff --git a/pyproject.toml b/pyproject.toml index 1b78697b8920..c3e51c2cf591 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,7 +74,7 @@ vllm = [ sglang = [ "uvloop", - "sglang[diffusion]==0.5.18", + "sglang[diffusion]==0.5.19", # sglang[diffusion] dropped accelerate in 0.5.12; diffusers still needs it. "accelerate>=0.17.0", "blake3>=1.0.0,<2.0.0", From 59fc9cb4baf6f236015bd314fada0062f3273c4a Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 1 Sep 2026 17:37:27 -0700 Subject: [PATCH 02/12] fix(sglang): support 0.5.19 API changes Signed-off-by: jain-ria --- components/src/dynamo/sglang/_compat.py | 92 +++++++++---- components/src/dynamo/sglang/args.py | 10 +- .../src/dynamo/sglang/init_multimodal.py | 2 + .../multimodal/encode_worker_handler.py | 125 ++++++++++++------ .../test_sglang_multimodal_embedding_cache.py | 26 ++-- .../tests/test_sglang_multimodal_video.py | 65 +++++++-- .../dynamo/sglang/tests/test_sglang_unit.py | 46 +++---- tests/report_pytest_markers.py | 1 + 8 files changed, 254 insertions(+), 113 deletions(-) diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 7eae14720108..1bd286f061ff 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -26,6 +26,44 @@ from functools import lru_cache, wraps from typing import Any +try: + from sglang.srt.utils.server_args_config_parser import ConfigArgumentMerger +except ImportError: + # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is 0.5.19+. + from sglang.srt.server_args_config_parser import ConfigArgumentMerger + +try: + from sglang.srt.arg_groups.model_override_base import ( + model_config_of as _sglang_model_config_of, + ) +except ImportError: + # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is 0.5.19+. + _sglang_model_config_of = None + +try: + from sglang.srt.runtime_context import publish as _sglang_publish +except ImportError: + # Fallback for SGLang 0.5.18 and the XPU 0.5.11 pin. Remove the 0.5.18 + # portion when minimum supported SGLang is 0.5.19+. + _sglang_publish = None + + +def model_config_of(server_args: Any) -> Any: + """Return the cached SGLang model config across 0.5.18 and 0.5.19.""" + legacy_get_model_config = getattr(server_args, "get_model_config", None) + if callable(legacy_get_model_config): + return legacy_get_model_config() + if _sglang_model_config_of is None: + raise AttributeError("SGLang does not expose a model-config accessor") + return _sglang_model_config_of(server_args) + + +def publish_server_args(server_args: Any, *, role: str) -> None: + """Publish process-wide SGLang configuration when the API is available.""" + if _sglang_publish is not None: + _sglang_publish(server_args, role=role) + + try: from sglang.srt.arg_groups.overrides import declare_late_resolution except ImportError: @@ -33,10 +71,19 @@ declare_late_resolution = None try: - from sglang.srt.arg_groups.overrides import resolved_view as sglang_resolved_view + from sglang.srt.arg_groups.model_override_base import ( + resolved_view as sglang_resolved_view, + ) except ImportError: - # SGLang #36255 exposes ServerArgs._resolved() instead. - sglang_resolved_view = None + # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is 0.5.19+. + try: + from sglang.srt.arg_groups.overrides import ( + resolved_view as sglang_resolved_view, + ) + except ImportError: + # The separately pinned XPU SGLang 0.5.11 stores effective values on + # ServerArgs directly. Remove when that pin is upgraded. + sglang_resolved_view = None logger = logging.getLogger(__name__) @@ -53,9 +100,10 @@ def _warn_require_reasoning_unsupported() -> None: def ensure_sglang_tensor_image_size() -> None: """Allow SGLang's image-token resolver to handle decoded image tensors. - SGLang 0.5.13 through 0.5.18 assume every decoded image exposes the PIL - ``height``/``width`` attributes. Its CUDA JPEG decoder instead returns a - CHW tensor, causing multimodal requests to fall back to retokenization. + SGLang 0.5.13 through the 0.5.19 release branch assume every decoded image + exposes the PIL ``height``/``width`` attributes. Its CUDA JPEG decoder + instead returns a CHW tensor, causing multimodal requests to fall back to + retokenization. Remove this compatibility override once the minimum supported SGLang release handles tensor image dimensions itself. @@ -98,26 +146,14 @@ def override_server_args(server_args: Any, source: str, **fields: Any) -> None: SGLang 0.5.18+ resolves its effective configuration separately from raw ``ServerArgs`` input. Declare pre-engine changes through its resolution API - so the engine's resolved projection observes them. SGLang 0.5.17 exposes - ``ServerArgs.override`` instead. The separately pinned XPU image still uses - SGLang 0.5.11, which predates both APIs; preserve its legacy assignment - behavior until its engine pin is upgraded. + so the engine's resolved projection observes them. The separately pinned + XPU image still uses SGLang 0.5.11, which predates that API; preserve its + legacy assignment behavior until its engine pin is upgraded. """ if declare_late_resolution is not None: declare_late_resolution(server_args, source, **fields) return - late_resolution = getattr(server_args, "_late_resolution", None) - if callable(late_resolution): - late_resolution(source, **fields) - return - - # Fallback for SGLang 0.5.17. Remove when minimum supported SGLang is 0.5.18+. - override = getattr(server_args, "override", None) - if callable(override): - override(source, **fields) - return - # XPU compatibility for SGLang 0.5.11. Remove when the XPU SGLang pin is # upgraded to 0.5.16+. for name, value in fields.items(): @@ -127,14 +163,11 @@ def override_server_args(server_args: Any, source: str, **fields: Any) -> None: def resolved_server_args(server_args: Any) -> Any: """Return SGLang's effective configuration for one initialized engine. - SGLang #36255 exposes ``ServerArgs._resolved()``. Current SGLang keeps - ``ServerArgs`` raw and exposes the same projection through - ``resolved_view()``. Older supported releases and Dynamo's non-LLM argument - stubs retain effective values on the object itself. + SGLang 0.5.18 and 0.5.19 keep ``ServerArgs`` raw and expose the effective + projection through ``resolved_view()``. The separately pinned XPU release + and Dynamo's non-LLM argument stubs retain effective values on the object + itself. """ - resolve = getattr(server_args, "_resolved", None) - if callable(resolve): - return resolve() if sglang_resolved_view is not None: return sglang_resolved_view(server_args) return server_args @@ -210,8 +243,11 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st __all__ = [ + "ConfigArgumentMerger", "ensure_sglang_tensor_image_size", "filter_supported_async_generate_kwargs", + "model_config_of", "override_server_args", + "publish_server_args", "require_reasoning_kwargs", ] diff --git a/components/src/dynamo/sglang/args.py b/components/src/dynamo/sglang/args.py index b24f8b48a356..f42ddfeb2b60 100644 --- a/components/src/dynamo/sglang/args.py +++ b/components/src/dynamo/sglang/args.py @@ -15,7 +15,6 @@ import yaml from sglang.srt.server_args import ServerArgs -from sglang.srt.server_args_config_parser import ConfigArgumentMerger from dynamo.common.config_dump import register_encoder from dynamo.common.configuration.groups import DynamoRuntimeConfig @@ -34,7 +33,12 @@ ) from dynamo.common.utils.runtime import parse_endpoint from dynamo.runtime.logging import configure_dynamo_logging -from dynamo.sglang._compat import ensure_sglang_tensor_image_size, resolved_server_args +from dynamo.sglang._compat import ( + ConfigArgumentMerger, + ensure_sglang_tensor_image_size, + model_config_of, + resolved_server_args, +) from dynamo.sglang.backend_args import DynamoSGLangArgGroup, DynamoSGLangConfig configure_dynamo_logging() @@ -621,7 +625,7 @@ async def parse_args(args: list[str]) -> Config: # Dynamo expects disjoint output_ids; ServerArgs is read-only after resolution. parsed_args.incremental_streaming_output = True server_args = ServerArgs.from_cli_args(parsed_args) - if server_args.get_model_config().is_multimodal: + if model_config_of(server_args).is_multimodal: ensure_sglang_tensor_image_size() if getattr(server_args, "schedule_low_priority_values_first", False): diff --git a/components/src/dynamo/sglang/init_multimodal.py b/components/src/dynamo/sglang/init_multimodal.py index 7fe77cc00c81..d1daa4fe0e3a 100644 --- a/components/src/dynamo/sglang/init_multimodal.py +++ b/components/src/dynamo/sglang/init_multimodal.py @@ -18,6 +18,7 @@ WorkerType, ) from dynamo.runtime import DistributedRuntime +from dynamo.sglang._compat import publish_server_args from dynamo.sglang.args import Config from dynamo.sglang.health_check import ( SglangDisaggHealthCheckPayload, @@ -60,6 +61,7 @@ async def init_multimodal_encode_worker( cache_publisher = MultimodalEmbeddingCachePublisher() await cache_publisher.create_endpoint(generate_endpoint) + publish_server_args(server_args, role="encoder") handler = MultimodalEncodeWorkerHandler( config, pd_worker_client, diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py index 075ed2f40cb9..826f45d00689 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py @@ -5,6 +5,7 @@ import importlib import json import logging +import uuid from dataclasses import dataclass from typing import Any, AsyncIterator, Dict, Optional from urllib.parse import urlparse @@ -15,7 +16,12 @@ # MMEncoder chain imports compiled CUDA ops; may fail in CPU-only environments. try: - from sglang.srt.disaggregation.encode_server import MMEncoder + try: + from sglang.srt.disaggregation.encoder.server import MMEncoder + except ImportError: + # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is + # 0.5.19+. + from sglang.srt.disaggregation.encode_server import MMEncoder from sglang.srt.managers.schedule_batch import Modality except (ImportError, OSError): MMEncoder = None # type: ignore[assignment] @@ -144,16 +150,15 @@ def _install_load_video_passthrough() -> None: Patch the name **as bound in each importing module**: they do ``from sglang.srt.utils import load_video``, so rebinding ``sglang.srt.utils.load_video`` alone leaves those call sites untouched. - ``encode_server`` is the one this handler actually goes through -- its - ``_flatten_and_load_videos`` calls its own binding -- and omitting it is why - an earlier revision still raised ``ValueError: Unsupported video input type`` - end to end while every unit test passed. - - Verified against ``v0.5.16``: ``encode_server``, ``base_processor`` and - ``utils`` are the ``srt`` modules that bind the name. Patch every one that - imports successfully and log which, so a future SGLang bump that moves the - call site shows up as a missing module rather than silently reverting to the - URL path. Idempotent; a no-op when SGLang is unavailable. + The encoder preprocessor is the one this handler actually goes through -- + its ``_flatten_and_load_videos`` calls its own binding -- and omitting it is + why an earlier revision still raised ``ValueError: Unsupported video input + type`` end to end while every unit test passed. + + SGLang 0.5.19 moved this binding from ``encode_server`` into + ``disaggregation.encoder.preprocessor``. Patch both locations so N and N-1 + remain supported, plus the shared processor and utils bindings. Idempotent; + a no-op when SGLang is unavailable. """ if not SGLANG_VIDEO_DECODER_AVAILABLE: return @@ -173,7 +178,8 @@ def _load_video_passthrough(video_file, *args, **kwargs): patched: list[str] = [] for module_path in ( - # The encode worker's own call site -- the one that matters here. + # Encoder call sites for SGLang 0.5.19 and 0.5.18, respectively. + "sglang.srt.disaggregation.encoder.preprocessor", "sglang.srt.disaggregation.encode_server", "sglang.srt.multimodal.processors.base_processor", "sglang.srt.utils", @@ -191,12 +197,17 @@ def _load_video_passthrough(video_file, *args, **kwargs): _wrap(module, orig) patched.append(module_path) - if "sglang.srt.disaggregation.encode_server" not in patched: + encoder_modules = { + "sglang.srt.disaggregation.encoder.preprocessor", + "sglang.srt.disaggregation.encode_server", + } + if not encoder_modules.intersection(patched): # Without this one the decoder reaches load_video unpatched and the # request fails with "Unsupported video input type" rather than falling # back, so make the gap visible instead of waiting for a 400. logger.warning( - "load_video passthrough not installed on encode_server (patched: %s); " + "load_video passthrough not installed on encoder preprocessor " + "(patched: %s); " "NVDEC video decoding will not work on this SGLang version", patched or "none", ) @@ -227,31 +238,35 @@ def _install_nvdec_video_metadata_shim() -> None: so the synthesized values below never apply to it. The shim remains only for genuine pre-decoded arrays. """ - try: - from sglang.srt.disaggregation import encode_server as es - except (ImportError, OSError): - return + for module_path in ( + "sglang.srt.disaggregation.encoder.preprocessor", + "sglang.srt.disaggregation.encode_server", + ): + try: + module = importlib.import_module(module_path) + except (ImportError, OSError): + continue - orig = getattr(es, "preprocess_video", None) - if orig is None or getattr(orig, "_dynamo_nvdec_shim", False): - return + orig = getattr(module, "preprocess_video", None) + if orig is None or getattr(orig, "_dynamo_nvdec_shim", False): + continue - async def _preprocess_video_with_metadata(vr, *args, **kwargs): - video, metadata = await orig(vr, *args, **kwargs) - if metadata is None and isinstance(video, np.ndarray) and video.ndim >= 1: - num_frames = int(video.shape[0]) - fps = _NVDEC_SHIM_FPS - metadata = { - "fps": fps, - "duration": (num_frames / fps) if fps else 0.0, - "total_num_frames": num_frames, - "frames_indices": list(range(num_frames)), - "video_backend": "nvdec", - } - return video, metadata + async def _preprocess_video_with_metadata(vr, *args, _orig=orig, **kwargs): + video, metadata = await _orig(vr, *args, **kwargs) + if metadata is None and isinstance(video, np.ndarray) and video.ndim >= 1: + num_frames = int(video.shape[0]) + fps = _NVDEC_SHIM_FPS + metadata = { + "fps": fps, + "duration": (num_frames / fps) if fps else 0.0, + "total_num_frames": num_frames, + "frames_indices": list(range(num_frames)), + "video_backend": "nvdec", + } + return video, metadata - _preprocess_video_with_metadata._dynamo_nvdec_shim = True # type: ignore[attr-defined] - es.preprocess_video = _preprocess_video_with_metadata + _preprocess_video_with_metadata._dynamo_nvdec_shim = True # type: ignore[attr-defined] + module.preprocess_video = _preprocess_video_with_metadata class MultimodalEncodeWorkerHandler(BaseWorkerHandler[SglangMultimodalRequest, str]): @@ -786,7 +801,7 @@ async def _encode_with_cache( encode_inputs = await self._build_encode_inputs( uncached_inputs, modality_name ) - grid_dim, new_embeddings, aux_data = await self.encoder._encode( + grid_dim, new_embeddings, aux_data = await self._encode_media( encode_inputs, modality ) # Verify SGLang output is on CPU as expected @@ -860,6 +875,40 @@ async def _encode_with_cache( full_embeddings = torch.cat(embedding_parts, dim=0) return torch.tensor(all_grid_thw), full_embeddings, all_entries + async def _encode_media( + self, media_inputs: list[Any], modality: Any + ) -> tuple[Any, torch.Tensor, dict[str, Any]]: + """Encode media across the SGLang 0.5.18 and 0.5.19 APIs.""" + legacy_encode = getattr(self.encoder, "_encode", None) + if callable(legacy_encode): + return await legacy_encode(media_inputs, modality) + + prepare = getattr(self.encoder, "_prepare_encode_context", None) + compute = getattr(self.encoder, "_compute_embedding", None) + if not callable(prepare) or not callable(compute): + raise AttributeError("SGLang MMEncoder does not expose an encode API") + + request = { + "req_id": f"dynamo-direct-{uuid.uuid4()}", + "num_parts": 1, + "part_idx": 0, + "mm_items": media_inputs, + "hashes": None, + } + encode_context = await prepare( + [request], + modality, + use_global_cache=False, + ) + embeddings = await compute(encode_context, keep_on_gpu=False) + if embeddings is None: + raise RuntimeError("SGLang MMEncoder returned no embeddings") + return ( + encode_context.preprocess_result.grid_thw, + embeddings, + encode_context.aux_data, + ) + def _extract_media_inputs( self, request: Dict[str, Any] ) -> tuple[list[Any], list[str]]: @@ -1120,7 +1169,7 @@ async def generate( encode_inputs = await self._build_encode_inputs( media_inputs, modality_name ) - grid_dim, embeddings, aux_data = await self.encoder._encode( + grid_dim, embeddings, aux_data = await self._encode_media( encode_inputs, modality_enum ) diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py index e99de7f35667..56524dcc1d7e 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py @@ -4,6 +4,7 @@ """Unit tests for SGLang multimodal embedding cache behavior.""" import asyncio +import importlib import json from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -975,8 +976,8 @@ async def _responses(): ) -def test_load_video_passthrough_patches_the_encode_server_binding() -> None: - """The passthrough must reach the binding encode_server actually calls. +def test_load_video_passthrough_patches_the_encoder_binding() -> None: + """The passthrough must reach the binding the encoder actually calls. Regression: an earlier revision patched base_processor and sglang.srt.utils but not encode_server, which does its own ``from sglang.srt.utils import @@ -984,10 +985,15 @@ def test_load_video_passthrough_patches_the_encode_server_binding() -> None: load_video was never reached -- while the deployed encode worker rejected each video request with "Unsupported video input type" and returned 400. """ - encode_server = pytest.importorskip( - "sglang.srt.disaggregation.encode_server", - reason="SGLang required to verify the patch target", - ) + try: + encoder_preprocessor = importlib.import_module( + "sglang.srt.disaggregation.encoder.preprocessor" + ) + except ImportError: + encoder_preprocessor = pytest.importorskip( + "sglang.srt.disaggregation.encode_server", + reason="SGLang required to verify the patch target", + ) from sglang.srt.utils.video_decoder import VideoDecoderWrapper from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import ( @@ -995,10 +1001,10 @@ def test_load_video_passthrough_patches_the_encode_server_binding() -> None: ) _install_load_video_passthrough() - patched = encode_server.load_video + patched = encoder_preprocessor.load_video assert getattr(patched, "_dynamo_nvdec_passthrough", False), ( - "encode_server.load_video is unpatched; NVDEC decoders will be rejected " - "by SGLang at request time" + "encoder preprocessor load_video is unpatched; NVDEC decoders will be " + "rejected by SGLang at request time" ) # A pre-built decoder passes straight through instead of raising. @@ -1015,4 +1021,4 @@ def test_load_video_passthrough_patches_the_encode_server_binding() -> None: patched(object()) _install_load_video_passthrough() # idempotent - assert encode_server.load_video is patched + assert encoder_preprocessor.load_video is patched diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index bae6b7a6b7be..ded986c7f92d 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -103,6 +103,44 @@ def test_extract_media_inputs_supports_mixed_image_and_video(): assert video_urls == ["https://example.com/clip.mp4"] +@pytest.mark.asyncio +async def test_encode_media_uses_sglang_0_5_19_pipeline(): + embeddings = torch.ones((4, 8)) + preprocess_result = SimpleNamespace(grid_thw=[[1, 2, 2]]) + encode_context = SimpleNamespace( + preprocess_result=preprocess_result, + aux_data={"example": "value"}, + ) + calls = [] + + class _Encoder: + async def _prepare_encode_context( + self, requests, modality, *, use_global_cache + ): + calls.append((requests, modality, use_global_cache)) + return encode_context + + async def _compute_embedding(self, context, *, keep_on_gpu): + assert context is encode_context + assert keep_on_gpu is False + return embeddings + + handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler) + handler.encoder = _Encoder() + + grid_thw, result, aux_data = await handler._encode_media( + ["https://example.com/image.png"], Modality.IMAGE + ) + + assert grid_thw == [[1, 2, 2]] + assert result is embeddings + assert aux_data == {"example": "value"} + requests, modality, use_global_cache = calls[0] + assert requests[0]["mm_items"] == ["https://example.com/image.png"] + assert modality == Modality.IMAGE + assert use_global_cache is False + + @pytest.mark.multimodal def test_extract_media_inputs_rejects_multimodal_cache_uuid(): handler = MultimodalEncodeWorkerHandler.__new__(MultimodalEncodeWorkerHandler) @@ -524,26 +562,35 @@ async def test_nvdec_video_metadata_shim_stamps_valid_metadata(): ndarray, and transformers >= 5.12 strict-rejects the resulting ``video_metadata=[None]``. ``_install_nvdec_video_metadata_shim`` wraps it so the ndarray carries a valid dict instead (validated end-to-end on gpu-ts - against the real ``MMEncoder._encode``: before FAIL -> after PASS). + against the real MMEncoder pipeline: before FAIL -> after PASS). """ - es = pytest.importorskip("sglang.srt.disaggregation.encode_server") + try: + encoder_preprocessor = importlib.import_module( + "sglang.srt.disaggregation.encoder.preprocessor" + ) + except ImportError: + encoder_preprocessor = pytest.importorskip( + "sglang.srt.disaggregation.encode_server" + ) from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import ( _NVDEC_SHIM_FPS, _install_nvdec_video_metadata_shim, ) - saved = es.preprocess_video + saved = encoder_preprocessor.preprocess_video try: _install_nvdec_video_metadata_shim() - assert getattr(es.preprocess_video, "_dynamo_nvdec_shim", False) + assert getattr( + encoder_preprocessor.preprocess_video, "_dynamo_nvdec_shim", False + ) # Idempotent: a second install must not double-wrap. - wrapped = es.preprocess_video + wrapped = encoder_preprocessor.preprocess_video _install_nvdec_video_metadata_shim() - assert es.preprocess_video is wrapped + assert encoder_preprocessor.preprocess_video is wrapped frames = np.zeros((5, 8, 8, 3), dtype=np.uint8) - video, meta = await es.preprocess_video(frames) + video, meta = await encoder_preprocessor.preprocess_video(frames) assert video is frames assert meta == { "fps": _NVDEC_SHIM_FPS, @@ -554,10 +601,10 @@ async def test_nvdec_video_metadata_shim_stamps_valid_metadata(): } # Non-ndarray inputs keep None metadata (the shim only touches pixels). - _, meta_none = await es.preprocess_video("not-an-array") + _, meta_none = await encoder_preprocessor.preprocess_video("not-an-array") assert meta_none is None finally: - es.preprocess_video = saved + encoder_preprocessor.preprocess_video = saved # --------------------------------------------------------------------------- diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py index 97a607c2a4eb..ca94553523d7 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_unit.py +++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py @@ -24,6 +24,7 @@ ensure_sglang_tensor_image_size, filter_supported_async_generate_kwargs, override_server_args, + publish_server_args, require_reasoning_kwargs, resolved_server_args, ) @@ -119,28 +120,6 @@ def declare(server_args, source, **fields): assert not hasattr(server_args, "enable_memory_saver") -def test_override_server_args_supports_sglang_0_5_17(monkeypatch): - calls = [] - - class ServerArgs: - def override(self, source, **fields): - calls.append((source, fields)) - for name, value in fields.items(): - object.__setattr__(self, name, value) - - monkeypatch.setattr(sglang_compat, "declare_late_resolution", None) - server_args = ServerArgs() - - override_server_args( - server_args, - "dynamo.test", - enable_memory_saver=True, - ) - - assert calls == [("dynamo.test", {"enable_memory_saver": True})] - assert server_args.enable_memory_saver is True - - def test_override_server_args_supports_legacy_xpu_pin(monkeypatch): monkeypatch.setattr(sglang_compat, "declare_late_resolution", None) server_args = SimpleNamespace(enable_memory_saver=False) @@ -156,6 +135,20 @@ def test_override_server_args_supports_legacy_xpu_pin(monkeypatch): assert server_args.load_format == "legacy-loader" +def test_publish_server_args_uses_runtime_context(monkeypatch): + calls = [] + server_args = SimpleNamespace() + monkeypatch.setattr( + sglang_compat, + "_sglang_publish", + lambda value, *, role: calls.append((value, role)), + ) + + publish_server_args(server_args, role="encoder") + + assert calls == [(server_args, "encoder")] + + def test_resolved_server_args_uses_declarative_view(monkeypatch): raw_server_args = SimpleNamespace(page_size=None) resolved_server_args_view = SimpleNamespace(page_size=64) @@ -170,12 +163,15 @@ def test_resolved_server_args_uses_declarative_view(monkeypatch): assert raw_server_args.page_size is None -def test_config_uses_resolved_server_args_after_runtime_init(): +def test_config_uses_resolved_server_args_after_runtime_init(monkeypatch): raw_server_args = SimpleNamespace(page_size=None, disaggregation_mode="null") resolved_server_args = SimpleNamespace(page_size=64, disaggregation_mode="null") - raw_server_args._resolved = lambda: resolved_server_args + monkeypatch.setattr( + sglang_compat, + "sglang_resolved_view", + lambda server_args: resolved_server_args, + ) config = sglang_args.Config(raw_server_args, SimpleNamespace()) - runtime_server_args = config.use_resolved_server_args(raw_server_args) assert raw_server_args.page_size is None diff --git a/tests/report_pytest_markers.py b/tests/report_pytest_markers.py index 0efdcbe2194c..fcdad9c0692f 100755 --- a/tests/report_pytest_markers.py +++ b/tests/report_pytest_markers.py @@ -180,6 +180,7 @@ "sglang.srt.utils", "sglang.srt.utils.hf_transformers_utils", "sglang.srt.utils.network", + "sglang.srt.utils.server_args_config_parser", "sglang.srt.utils.video_decoder", "sglang.srt.disaggregation", "sglang.srt.disaggregation.kv_events", From 1b7e2d2923f345dab24e8eb815d9aee3b3defbc7 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 1 Sep 2026 23:59:09 -0700 Subject: [PATCH 03/12] fix(sglang): centralize encoder compatibility Signed-off-by: jain-ria --- components/src/dynamo/sglang/AGENTS.md | 9 +- components/src/dynamo/sglang/_compat.py | 79 +++++++++++++++- .../multimodal/encode_worker_handler.py | 94 ++++++------------- 3 files changed, 111 insertions(+), 71 deletions(-) diff --git a/components/src/dynamo/sglang/AGENTS.md b/components/src/dynamo/sglang/AGENTS.md index d35048a72096..b7139517a4e0 100644 --- a/components/src/dynamo/sglang/AGENTS.md +++ b/components/src/dynamo/sglang/AGENTS.md @@ -70,10 +70,11 @@ Worker dispatch (main.py:60-132): have `max_running_requests`, `dllm_algorithm_config`, or other LLM-specific fields. Use `getattr()` when accessing fields that may not exist on the stub. -SGLang 0.5.17 makes a resolved `ServerArgs` unconditionally read-only. Apply Dynamo's -post-resolution startup overrides through `_compat.override_server_args()`; control-plane -updates after engine creation should use the tokenizer manager's update API instead of -assigning fields on `server_args`. +The supported SGLang 0.5.18/0.5.19 releases keep raw input on `ServerArgs` and +publish the resolved configuration separately. Apply Dynamo's post-resolution startup +overrides through `_compat.override_server_args()`; control-plane updates after engine +creation should use the tokenizer manager's update API instead of assigning fields on +`server_args`. **DynamoConfig** combines `DynamoRuntimeConfig` (common flags like `--namespace`, `--output-modalities`, `--media-output-fs-url`) with `DynamoSGLangConfig` (sglang-specific diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 1bd286f061ff..3a7d6900ac4f 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -20,10 +20,13 @@ >= 0.5.11. Pass through; do not re-encode. """ +import importlib import inspect import logging +import uuid from collections.abc import Mapping from functools import lru_cache, wraps +from types import ModuleType from typing import Any try: @@ -67,7 +70,8 @@ def publish_server_args(server_args: Any, *, role: str) -> None: try: from sglang.srt.arg_groups.overrides import declare_late_resolution except ImportError: - # SGLang 0.5.17 and the XPU 0.5.11 pin predate declarations. + # The separately pinned XPU SGLang 0.5.11 predates declarations. Remove + # when the XPU SGLang pin is upgraded to 0.5.18+. declare_late_resolution = None try: @@ -88,6 +92,76 @@ def publish_server_args(server_args: Any, *, role: str) -> None: logger = logging.getLogger(__name__) +def get_mm_encoder_class() -> type[Any]: + """Load MMEncoder from the supported SGLang package layout. + + Keep this import deferred because the encoder module imports compiled CUDA + operators and this compatibility module is also collected on CPU-only CI + hosts. + """ + try: + from sglang.srt.disaggregation.encoder.server import MMEncoder + except ImportError: + # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is + # 0.5.19+. + from sglang.srt.disaggregation.encode_server import MMEncoder + + return MMEncoder + + +def get_encoder_preprocessor_modules() -> tuple[ModuleType, ...]: + """Return importable encoder modules that bind video preprocessing APIs.""" + modules: list[ModuleType] = [] + for module_path in ( + "sglang.srt.disaggregation.encoder.preprocessor", + # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is + # 0.5.19+. + "sglang.srt.disaggregation.encode_server", + ): + try: + modules.append(importlib.import_module(module_path)) + except (ImportError, OSError): + continue + return tuple(modules) + + +async def mm_encode( + encoder: Any, media_inputs: list[Any], modality: Any +) -> tuple[Any, Any, dict[str, Any]]: + """Encode media across the supported SGLang MMEncoder APIs.""" + legacy_encode = getattr(encoder, "_encode", None) + if callable(legacy_encode): + # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is + # 0.5.19+. + return await legacy_encode(media_inputs, modality) + + prepare = getattr(encoder, "_prepare_encode_context", None) + compute = getattr(encoder, "_compute_embedding", None) + if not callable(prepare) or not callable(compute): + raise RuntimeError("SGLang MMEncoder does not expose an encode API") + + request = { + "req_id": f"dynamo-direct-{uuid.uuid4()}", + "num_parts": 1, + "part_idx": 0, + "mm_items": media_inputs, + "hashes": None, + } + encode_context = await prepare( + [request], + modality, + use_global_cache=False, + ) + embeddings = await compute(encode_context, keep_on_gpu=False) + if embeddings is None: + raise RuntimeError("SGLang MMEncoder returned no embeddings") + return ( + encode_context.preprocess_result.grid_thw, + embeddings, + encode_context.aux_data, + ) + + @lru_cache(maxsize=1) def _warn_require_reasoning_unsupported() -> None: logger.warning( @@ -246,6 +320,9 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st "ConfigArgumentMerger", "ensure_sglang_tensor_image_size", "filter_supported_async_generate_kwargs", + "get_encoder_preprocessor_modules", + "get_mm_encoder_class", + "mm_encode", "model_config_of", "override_server_args", "publish_server_args", diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py index 826f45d00689..310c1cd5da0c 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py @@ -5,7 +5,6 @@ import importlib import json import logging -import uuid from dataclasses import dataclass from typing import Any, AsyncIterator, Dict, Optional from urllib.parse import urlparse @@ -14,17 +13,11 @@ import torch from blake3 import blake3 -# MMEncoder chain imports compiled CUDA ops; may fail in CPU-only environments. +# Modality is safe to import during collection; MMEncoder itself is loaded +# lazily by get_mm_encoder_class because it imports compiled CUDA operators. try: - try: - from sglang.srt.disaggregation.encoder.server import MMEncoder - except ImportError: - # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is - # 0.5.19+. - from sglang.srt.disaggregation.encode_server import MMEncoder from sglang.srt.managers.schedule_batch import Modality except (ImportError, OSError): - MMEncoder = None # type: ignore[assignment] Modality = None # type: ignore[assignment] from sglang.srt.parser.conversation import chat_templates from transformers import AutoTokenizer @@ -58,6 +51,11 @@ from dynamo.common.utils import nvtx_utils as _nvtx from dynamo.common.utils.env import env_bool from dynamo.llm import MultimodalEmbeddingCachePublisher +from dynamo.sglang._compat import ( + get_encoder_preprocessor_modules, + get_mm_encoder_class, + mm_encode, +) from dynamo.sglang.args import Config from dynamo.sglang.protocol import ( MultiModalGroup, @@ -155,10 +153,9 @@ def _install_load_video_passthrough() -> None: why an earlier revision still raised ``ValueError: Unsupported video input type`` end to end while every unit test passed. - SGLang 0.5.19 moved this binding from ``encode_server`` into - ``disaggregation.encoder.preprocessor``. Patch both locations so N and N-1 - remain supported, plus the shared processor and utils bindings. Idempotent; - a no-op when SGLang is unavailable. + The compatibility shim supplies the encoder module for N and N-1. Patch it + plus the shared processor and utils bindings. Idempotent; a no-op when + SGLang is unavailable. """ if not SGLANG_VIDEO_DECODER_AVAILABLE: return @@ -176,18 +173,20 @@ def _load_video_passthrough(video_file, *args, **kwargs): _load_video_passthrough._dynamo_nvdec_passthrough = True # type: ignore[attr-defined] module.load_video = _load_video_passthrough - patched: list[str] = [] + encoder_modules = get_encoder_preprocessor_modules() + modules = list(encoder_modules) for module_path in ( - # Encoder call sites for SGLang 0.5.19 and 0.5.18, respectively. - "sglang.srt.disaggregation.encoder.preprocessor", - "sglang.srt.disaggregation.encode_server", "sglang.srt.multimodal.processors.base_processor", "sglang.srt.utils", ): try: - module = importlib.import_module(module_path) + modules.append(importlib.import_module(module_path)) except (ImportError, OSError): continue + + patched: list[str] = [] + for module in modules: + module_path = module.__name__ orig = getattr(module, "load_video", None) if orig is None: continue @@ -197,11 +196,8 @@ def _load_video_passthrough(video_file, *args, **kwargs): _wrap(module, orig) patched.append(module_path) - encoder_modules = { - "sglang.srt.disaggregation.encoder.preprocessor", - "sglang.srt.disaggregation.encode_server", - } - if not encoder_modules.intersection(patched): + encoder_module_names = {module.__name__ for module in encoder_modules} + if not encoder_module_names.intersection(patched): # Without this one the decoder reaches load_video unpatched and the # request fails with "Unsupported video input type" rather than falling # back, so make the gap visible instead of waiting for a 400. @@ -238,15 +234,7 @@ def _install_nvdec_video_metadata_shim() -> None: so the synthesized values below never apply to it. The shim remains only for genuine pre-decoded arrays. """ - for module_path in ( - "sglang.srt.disaggregation.encoder.preprocessor", - "sglang.srt.disaggregation.encode_server", - ): - try: - module = importlib.import_module(module_path) - except (ImportError, OSError): - continue - + for module in get_encoder_preprocessor_modules(): orig = getattr(module, "preprocess_video", None) if orig is None or getattr(orig, "_dynamo_nvdec_shim", False): continue @@ -266,7 +254,7 @@ async def _preprocess_video_with_metadata(vr, *args, _orig=orig, **kwargs): return video, metadata _preprocess_video_with_metadata._dynamo_nvdec_shim = True # type: ignore[attr-defined] - module.preprocess_video = _preprocess_video_with_metadata + module.preprocess_video = _preprocess_video_with_metadata # type: ignore[attr-defined] class MultimodalEncodeWorkerHandler(BaseWorkerHandler[SglangMultimodalRequest, str]): @@ -307,15 +295,17 @@ def __init__( self.num_video_frames = max(1, VideoLoader.NUM_FRAMES_DEFAULT) self._url_policy = UrlValidationPolicy.from_env() - if MMEncoder is None: + try: + mm_encoder_class = get_mm_encoder_class() + except (ImportError, OSError) as exc: raise RuntimeError( "MMEncoder is not available. " "Multimodal encode worker requires a CUDA environment." - ) + ) from exc # torch.distributed requires a dist_init_method even for tp=1; # port 0 lets the OS assign a free port. - self.encoder = MMEncoder( + self.encoder = mm_encoder_class( server_args=config.server_args, dist_init_method="tcp://127.0.0.1:0", rank=0, @@ -878,36 +868,8 @@ async def _encode_with_cache( async def _encode_media( self, media_inputs: list[Any], modality: Any ) -> tuple[Any, torch.Tensor, dict[str, Any]]: - """Encode media across the SGLang 0.5.18 and 0.5.19 APIs.""" - legacy_encode = getattr(self.encoder, "_encode", None) - if callable(legacy_encode): - return await legacy_encode(media_inputs, modality) - - prepare = getattr(self.encoder, "_prepare_encode_context", None) - compute = getattr(self.encoder, "_compute_embedding", None) - if not callable(prepare) or not callable(compute): - raise AttributeError("SGLang MMEncoder does not expose an encode API") - - request = { - "req_id": f"dynamo-direct-{uuid.uuid4()}", - "num_parts": 1, - "part_idx": 0, - "mm_items": media_inputs, - "hashes": None, - } - encode_context = await prepare( - [request], - modality, - use_global_cache=False, - ) - embeddings = await compute(encode_context, keep_on_gpu=False) - if embeddings is None: - raise RuntimeError("SGLang MMEncoder returned no embeddings") - return ( - encode_context.preprocess_result.grid_thw, - embeddings, - encode_context.aux_data, - ) + """Encode media through Dynamo's SGLang compatibility boundary.""" + return await mm_encode(self.encoder, media_inputs, modality) def _extract_media_inputs( self, request: Dict[str, Any] From b668e4cfdc8c199915498a1f158055e49c8bf820 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Wed, 2 Sep 2026 13:02:47 -0700 Subject: [PATCH 04/12] fix(sglang): preserve config parser import guard Signed-off-by: jain-ria --- components/src/dynamo/sglang/_compat.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 20ddc9c5b64b..be4168d40bdb 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -31,8 +31,10 @@ try: from sglang.srt.utils.server_args_config_parser import ConfigArgumentMerger -except ImportError: - # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is 0.5.19+. +except ModuleNotFoundError as exc: + if exc.name != "sglang.srt.utils.server_args_config_parser": + raise + # Keep the CUDA 0.5.18 and XPU 0.5.11 pins working until both move here. from sglang.srt.server_args_config_parser import ConfigArgumentMerger try: From cbd126efe20d35140659c16b5599ff200528d266 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Thu, 3 Sep 2026 11:20:14 -0700 Subject: [PATCH 05/12] fix(sglang): preserve merged accessor compatibility Signed-off-by: jain-ria --- components/src/dynamo/sglang/_compat.py | 62 ++++++++++++------- components/src/dynamo/sglang/args.py | 4 +- .../dynamo/sglang/tests/test_sglang_unit.py | 14 ++--- 3 files changed, 48 insertions(+), 32 deletions(-) diff --git a/components/src/dynamo/sglang/_compat.py b/components/src/dynamo/sglang/_compat.py index 81f92448b9ca..090e90ed5937 100644 --- a/components/src/dynamo/sglang/_compat.py +++ b/components/src/dynamo/sglang/_compat.py @@ -38,16 +38,22 @@ from sglang.srt.server_args_config_parser import ConfigArgumentMerger try: - from sglang.srt.arg_groups.model_override_base import ( - model_config_of as _sglang_model_config_of, + from sglang.srt.arg_groups.overrides import ( + model_config_of as sglang_model_config_of, ) - from sglang.srt.arg_groups.model_override_base import ( - use_mla_backend as _sglang_use_mla_backend, +except ImportError: + # Fallback for sglang <= 0.5.18, which exposes ServerArgs.get_model_config(). + # Remove when min supported version has the accessor move (sgl #36972). + sglang_model_config_of = None + +try: + from sglang.srt.arg_groups.overrides import ( + use_mla_backend as sglang_use_mla_backend, ) except ImportError: - # Fallback for SGLang 0.5.18. Remove when minimum supported SGLang is 0.5.19+. - _sglang_model_config_of = None - _sglang_use_mla_backend = None + # Fallback for sglang <= 0.5.18, which exposes ServerArgs.use_mla_backend(). + # Remove when min supported version has the accessor move (sgl #36972). + sglang_use_mla_backend = None try: from sglang.srt.runtime_context import publish as _sglang_publish @@ -57,24 +63,34 @@ _sglang_publish = None -def model_config_of(server_args: Any) -> Any: - """Return the cached SGLang model config across 0.5.18 and 0.5.19.""" - legacy_get_model_config = getattr(server_args, "get_model_config", None) - if callable(legacy_get_model_config): - return legacy_get_model_config() - if _sglang_model_config_of is None: - raise AttributeError("SGLang does not expose a model-config accessor") - return _sglang_model_config_of(server_args) +def get_sglang_model_config(server_args: Any) -> Any: + """Return the resolved model config across SGLang ServerArgs APIs. + + SGLang #36972 moved ``ServerArgs.get_model_config()`` to the module-level + ``model_config_of()``. Remove the legacy branch when the minimum supported + SGLang release contains that move. + """ + legacy_getter = getattr(server_args, "get_model_config", None) + if legacy_getter is not None: + return legacy_getter() + if sglang_model_config_of is None: + raise AttributeError("SGLang does not expose a model config accessor") + return sglang_model_config_of(server_args) def sglang_uses_mla_backend(server_args: Any) -> bool: - """Return whether SGLang selected an MLA attention backend across versions.""" - legacy_use_mla_backend = getattr(server_args, "use_mla_backend", None) - if callable(legacy_use_mla_backend): - return bool(legacy_use_mla_backend()) - if _sglang_use_mla_backend is None: - raise AttributeError("SGLang does not expose an MLA-backend accessor") - return bool(_sglang_use_mla_backend(server_args)) + """Return whether this configuration selects SGLang's MLA attention backend. + + SGLang #36972 moved ``ServerArgs.use_mla_backend()`` to the module-level + ``use_mla_backend()``. Remove the legacy branch when the minimum supported + SGLang release contains that move. + """ + legacy_getter = getattr(server_args, "use_mla_backend", None) + if legacy_getter is not None: + return bool(legacy_getter()) + if sglang_use_mla_backend is None: + raise AttributeError("SGLang does not expose an MLA backend accessor") + return bool(sglang_use_mla_backend(server_args)) def publish_server_args(server_args: Any, *, role: str) -> None: @@ -338,8 +354,8 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st "filter_supported_async_generate_kwargs", "get_encoder_preprocessor_modules", "get_mm_encoder_class", + "get_sglang_model_config", "mm_encode", - "model_config_of", "override_server_args", "publish_server_args", "require_reasoning_kwargs", diff --git a/components/src/dynamo/sglang/args.py b/components/src/dynamo/sglang/args.py index f42ddfeb2b60..a3fe2882786e 100644 --- a/components/src/dynamo/sglang/args.py +++ b/components/src/dynamo/sglang/args.py @@ -36,7 +36,7 @@ from dynamo.sglang._compat import ( ConfigArgumentMerger, ensure_sglang_tensor_image_size, - model_config_of, + get_sglang_model_config, resolved_server_args, ) from dynamo.sglang.backend_args import DynamoSGLangArgGroup, DynamoSGLangConfig @@ -625,7 +625,7 @@ async def parse_args(args: list[str]) -> Config: # Dynamo expects disjoint output_ids; ServerArgs is read-only after resolution. parsed_args.incremental_streaming_output = True server_args = ServerArgs.from_cli_args(parsed_args) - if model_config_of(server_args).is_multimodal: + if get_sglang_model_config(server_args).is_multimodal: ensure_sglang_tensor_image_size() if getattr(server_args, "schedule_low_priority_values_first", False): diff --git a/components/src/dynamo/sglang/tests/test_sglang_unit.py b/components/src/dynamo/sglang/tests/test_sglang_unit.py index 2d7438362d50..1d59854237cd 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_unit.py +++ b/components/src/dynamo/sglang/tests/test_sglang_unit.py @@ -23,7 +23,7 @@ from dynamo.sglang._compat import ( ensure_sglang_tensor_image_size, filter_supported_async_generate_kwargs, - model_config_of, + get_sglang_model_config, override_server_args, publish_server_args, require_reasoning_kwargs, @@ -170,11 +170,11 @@ def test_compat_uses_current_sglang_model_config_accessor(monkeypatch): server_args = SimpleNamespace() monkeypatch.setattr( sglang_compat, - "_sglang_model_config_of", + "sglang_model_config_of", lambda value: expected if value is server_args else None, ) - assert model_config_of(server_args) is expected + assert get_sglang_model_config(server_args) is expected def test_compat_uses_legacy_sglang_model_config_accessor(monkeypatch): @@ -182,18 +182,18 @@ def test_compat_uses_legacy_sglang_model_config_accessor(monkeypatch): server_args = SimpleNamespace(get_model_config=lambda: expected) monkeypatch.setattr( sglang_compat, - "_sglang_model_config_of", + "sglang_model_config_of", lambda _: pytest.fail("current accessor should not run for legacy ServerArgs"), ) - assert model_config_of(server_args) is expected + assert get_sglang_model_config(server_args) is expected def test_compat_uses_current_sglang_mla_accessor(monkeypatch): server_args = SimpleNamespace() monkeypatch.setattr( sglang_compat, - "_sglang_use_mla_backend", + "sglang_use_mla_backend", lambda value: value is server_args, ) @@ -204,7 +204,7 @@ def test_compat_uses_legacy_sglang_mla_accessor(monkeypatch): server_args = SimpleNamespace(use_mla_backend=lambda: True) monkeypatch.setattr( sglang_compat, - "_sglang_use_mla_backend", + "sglang_use_mla_backend", lambda _: pytest.fail("current accessor should not run for legacy ServerArgs"), ) From c7ffadcb09b41ae78843880f5889cf91b8377785 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 8 Sep 2026 16:16:19 -0700 Subject: [PATCH 06/12] build(sglang): use 0.5.19 release artifacts Signed-off-by: jain-ria --- container/compliance/base_sboms/manifest.json | 14 +++++++------- container/context.yaml | 2 +- lib/sidecar/sglang/README.md | 2 +- lib/sidecar/sglang/deploy/agg_kv_router.yaml | 2 +- lib/sidecar/sglang/deploy/disagg_kv_router.yaml | 4 ++-- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/container/compliance/base_sboms/manifest.json b/container/compliance/base_sboms/manifest.json index 5bc5a7f9402a..5b1a43a33906 100644 --- a/container/compliance/base_sboms/manifest.json +++ b/container/compliance/base_sboms/manifest.json @@ -110,10 +110,10 @@ "baseline_image": "nvidia/cuda", "baseline_sbom": "cuda@0230b7f2-amd64.cdx.json", "baseline_tag": "13.0.3-cudnn-devel-ubuntu24.04", - "capture_note": "lmsysorg/sglang is built on this nvidia/cuda cudnn-devel base; the vendor image layout is not layer-prefix preserving, so layer-prefix validation was skipped. All from-image delta components passed policy validation at capture.", - "from_digest": "sha256:d89bf566cec87e4beed8ee4068b409552eef1f83d3d9c9efb94a6e98753a70cd", + "capture_note": "lmsysorg/sglang is built on this nvidia/cuda cudnn-devel base; the vendor image layout is not layer-prefix preserving, so layer-prefix validation was skipped. Registry digest and CUDA 13.0.3 metadata were verified locally; full from-image delta policy validation runs in CI.", + "from_digest": "sha256:710bc11443a7b1807d69803386468101bcfced35f86bf8fe92a8209e05a2f052", "from_image": "lmsysorg/sglang", - "from_tag": "v0.5.18-cu130-runtime", + "from_tag": "v0.5.19-cu130-runtime", "layer_prefix_check_skipped": true, "platform": "linux/amd64" }, @@ -122,10 +122,10 @@ "baseline_image": "nvidia/cuda", "baseline_sbom": "cuda@0230b7f2-arm64.cdx.json", "baseline_tag": "13.0.3-cudnn-devel-ubuntu24.04", - "capture_note": "lmsysorg/sglang is built on this nvidia/cuda cudnn-devel base; the vendor image layout is not layer-prefix preserving, so layer-prefix validation was skipped. All from-image delta components passed policy validation at capture.", - "from_digest": "sha256:d89bf566cec87e4beed8ee4068b409552eef1f83d3d9c9efb94a6e98753a70cd", + "capture_note": "lmsysorg/sglang is built on this nvidia/cuda cudnn-devel base; the vendor image layout is not layer-prefix preserving, so layer-prefix validation was skipped. Registry digest and CUDA 13.0.3 metadata were verified locally; full from-image delta policy validation runs in CI.", + "from_digest": "sha256:710bc11443a7b1807d69803386468101bcfced35f86bf8fe92a8209e05a2f052", "from_image": "lmsysorg/sglang", - "from_tag": "v0.5.18-cu130-runtime", + "from_tag": "v0.5.19-cu130-runtime", "layer_prefix_check_skipped": true, "platform": "linux/arm64" }, @@ -162,6 +162,6 @@ "platform": "platform pinned for layer-prefix verification, e.g. linux/amd64" }, "format": "CycloneDX 1.6 JSON, slim filter (drop properties/hashes/dependencies; keep evidence). Hard cap 5 MB per file.", - "generated_at": "2026-09-01T00:35:32+00:00", + "generated_at": "2026-09-08T22:54:27+00:00", "schema_version": 2 } diff --git a/container/context.yaml b/container/context.yaml index 685930a4c3e4..5b2c03db3f72 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -123,7 +123,7 @@ sglang: base_image: nvcr.io/nvidia/cuda-dl-base runtime_image: lmsysorg/sglang base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04 - runtime_image_tag: v0.5.18-cu130-runtime + runtime_image_tag: v0.5.19-cu130-runtime # Baseline is the TRUE base of the third-party lmsysorg/sglang image, # nvidia/cuda:13.0.3-cudnn-devel-ubuntu24.04 (Docker Hub; CUDA_VERSION=13.0.3, # ubuntu24.04), NOT a self-baseline of lmsysorg/sglang. diff --git a/lib/sidecar/sglang/README.md b/lib/sidecar/sglang/README.md index faf4a528a128..cba07fd3c117 100644 --- a/lib/sidecar/sglang/README.md +++ b/lib/sidecar/sglang/README.md @@ -77,7 +77,7 @@ command. > The engine image must be a stock SGLang **v0.5.16+** build: the native gRPC > server (`--grpc-port`) landed there. The KV-routing examples require > **v0.5.18+** because the sidecar discovers their structured KV-event -> descriptor through `GetServerInfo`. They use `lmsysorg/sglang:v0.5.18`. +> descriptor through `GetServerInfo`. They use `lmsysorg/sglang:v0.5.19`. ### Prerequisites diff --git a/lib/sidecar/sglang/deploy/agg_kv_router.yaml b/lib/sidecar/sglang/deploy/agg_kv_router.yaml index 85f140f6d4c9..6c3d82d97405 100644 --- a/lib/sidecar/sglang/deploy/agg_kv_router.yaml +++ b/lib/sidecar/sglang/deploy/agg_kv_router.yaml @@ -42,7 +42,7 @@ spec: # separate pod network namespace. initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.18 + image: lmsysorg/sglang:v0.5.19 restartPolicy: Always startupProbe: exec: diff --git a/lib/sidecar/sglang/deploy/disagg_kv_router.yaml b/lib/sidecar/sglang/deploy/disagg_kv_router.yaml index dd4b6a5756d8..a30165d75848 100644 --- a/lib/sidecar/sglang/deploy/disagg_kv_router.yaml +++ b/lib/sidecar/sglang/deploy/disagg_kv_router.yaml @@ -50,7 +50,7 @@ spec: sizeLimit: 10Gi initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.18 + image: lmsysorg/sglang:v0.5.19 restartPolicy: Always securityContext: capabilities: @@ -144,7 +144,7 @@ spec: sizeLimit: 10Gi initContainers: - name: sglang-engine - image: lmsysorg/sglang:v0.5.18 + image: lmsysorg/sglang:v0.5.19 restartPolicy: Always securityContext: capabilities: From afab7a353b9e1fa664c10b8e7a0c77a0a53fe7e4 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 8 Sep 2026 16:32:00 -0700 Subject: [PATCH 07/12] fix(sidecar): include inference gateway crate in build context Signed-off-by: jain-ria --- lib/sidecar/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/sidecar/Dockerfile b/lib/sidecar/Dockerfile index 7443404aa2a7..231e7b08b5f4 100644 --- a/lib/sidecar/Dockerfile +++ b/lib/sidecar/Dockerfile @@ -73,6 +73,7 @@ COPY .cargo/ .cargo/ COPY lib/ lib/ COPY examples/router/custom-policy-example/ examples/router/custom-policy-example/ COPY deploy/inference-gateway/ext-proc/ deploy/inference-gateway/ext-proc/ +COPY deploy/inference-gateway/sidecar/ deploy/inference-gateway/sidecar/ # ---- vLLM builder ----------------------------------------------------------- FROM builder-base AS vllm-builder From a05d900acf32d146fdf4a566536ddea021c986db Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 8 Sep 2026 16:43:34 -0700 Subject: [PATCH 08/12] fix(sglang): seed dev venv from upstream runtime Signed-off-by: jain-ria --- container/templates/dev.Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/container/templates/dev.Dockerfile b/container/templates/dev.Dockerfile index 6407b25e151e..7ab055b0e726 100644 --- a/container/templates/dev.Dockerfile +++ b/container/templates/dev.Dockerfile @@ -355,11 +355,13 @@ RUN cp /tmp/uv-binary ${VIRTUAL_ENV}/bin/uv && \ chmod +x ${VIRTUAL_ENV}/bin/uv && \ pip install maturin[patchelf] {% else %} -# SGLang CUDA: Create venv with --system-site-packages to inherit runtime packages +# SGLang CUDA: Create a writable Dynamo venv and seed it from the upstream +# SGLang venv. The 0.5.19 runtime moved its packages from the system Python's +# dist-packages directory to /opt/sglang. COPY --from=ghcr.io/astral-sh/uv:{{ context.dynamo.uv_version }} /uv /tmp/uv-binary RUN mkdir -p /opt/dynamo/venv && \ python3 -m venv --system-site-packages /opt/dynamo/venv && \ - cp -r /usr/local/lib/python${PYTHON_VERSION}/dist-packages/* \ + cp -r /opt/sglang/lib/python${PYTHON_VERSION}/site-packages/. \ /opt/dynamo/venv/lib/python${PYTHON_VERSION}/site-packages/ && \ chmod -R g+w /opt/dynamo/venv/lib/python${PYTHON_VERSION}/site-packages/ && \ cp /tmp/uv-binary /opt/dynamo/venv/bin/uv && \ From 264839fb9f0f1295ed29662e37cf211fd725af28 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 8 Sep 2026 17:27:00 -0700 Subject: [PATCH 09/12] chore(sglang): address review feedback Signed-off-by: jain-ria --- .../multimodal/encode_worker_handler.py | 12 +++--------- .../tests/test_sglang_multimodal_embedding_cache.py | 5 +---- .../sglang/tests/test_sglang_multimodal_video.py | 7 ++----- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py index 310c1cd5da0c..42209b9edd90 100644 --- a/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py +++ b/components/src/dynamo/sglang/request_handlers/multimodal/encode_worker_handler.py @@ -148,14 +148,9 @@ def _install_load_video_passthrough() -> None: Patch the name **as bound in each importing module**: they do ``from sglang.srt.utils import load_video``, so rebinding ``sglang.srt.utils.load_video`` alone leaves those call sites untouched. - The encoder preprocessor is the one this handler actually goes through -- - its ``_flatten_and_load_videos`` calls its own binding -- and omitting it is - why an earlier revision still raised ``ValueError: Unsupported video input - type`` end to end while every unit test passed. - - The compatibility shim supplies the encoder module for N and N-1. Patch it - plus the shared processor and utils bindings. Idempotent; a no-op when - SGLang is unavailable. + The encoder preprocessor calls its own ``load_video`` binding, so patch that + module as well as the shared processor and utils bindings. Idempotent; a + no-op when SGLang is unavailable. """ if not SGLANG_VIDEO_DECODER_AVAILABLE: return @@ -868,7 +863,6 @@ async def _encode_with_cache( async def _encode_media( self, media_inputs: list[Any], modality: Any ) -> tuple[Any, torch.Tensor, dict[str, Any]]: - """Encode media through Dynamo's SGLang compatibility boundary.""" return await mm_encode(self.encoder, media_inputs, modality) def _extract_media_inputs( diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py index 56524dcc1d7e..4d53e0a7d1bc 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_embedding_cache.py @@ -31,6 +31,7 @@ from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import ( Modality, MultimodalEncodeWorkerHandler, + _install_load_video_passthrough, ) pytestmark = [ @@ -996,10 +997,6 @@ def test_load_video_passthrough_patches_the_encoder_binding() -> None: ) from sglang.srt.utils.video_decoder import VideoDecoderWrapper - from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import ( - _install_load_video_passthrough, - ) - _install_load_video_passthrough() patched = encoder_preprocessor.load_video assert getattr(patched, "_dynamo_nvdec_passthrough", False), ( diff --git a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py index ad927ad2a065..3631a96335f5 100644 --- a/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py +++ b/components/src/dynamo/sglang/tests/test_sglang_multimodal_video.py @@ -22,8 +22,10 @@ StopConditions, ) from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import ( + _NVDEC_SHIM_FPS, Modality, MultimodalEncodeWorkerHandler, + _install_nvdec_video_metadata_shim, ) from dynamo.sglang.request_handlers.multimodal.worker_handler import ( EmbeddingsProcessor, @@ -577,11 +579,6 @@ async def test_nvdec_video_metadata_shim_stamps_valid_metadata(): encoder_preprocessor = pytest.importorskip( "sglang.srt.disaggregation.encode_server" ) - from dynamo.sglang.request_handlers.multimodal.encode_worker_handler import ( - _NVDEC_SHIM_FPS, - _install_nvdec_video_metadata_shim, - ) - saved = encoder_preprocessor.preprocess_video try: _install_nvdec_video_metadata_shim() From 29df7e4d8fdea09ccdbce05bb492c9033fde8080 Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 8 Sep 2026 19:06:21 -0700 Subject: [PATCH 10/12] test(sglang): allow optional decoder install in test image Signed-off-by: jain-ria --- container/Dockerfile.test | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/container/Dockerfile.test b/container/Dockerfile.test index bf71afcd34ff..79f5bcd28c40 100644 --- a/container/Dockerfile.test +++ b/container/Dockerfile.test @@ -33,6 +33,15 @@ RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/ --requirement /tmp/requirements.test.txt; \ fi +# Some serve tests add an optional media decoder immediately before launching +# the backend. GitHub's ARC runner executes this image as an arbitrary uid, so +# make only the test image's venv package directory writable by that uid. The +# shipped runtime image remains read-only. +RUN if [ -n "${VIRTUAL_ENV:-}" ]; then \ + SITE_PACKAGES="$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" && \ + chmod o+w "${SITE_PACKAGES}"; \ + fi + # Copy pytest configuration (markers, filterwarnings, asyncio_mode, etc.) COPY --chmod=664 --chown=dynamo:0 pyproject.toml /workspace/pyproject.toml COPY --chmod=664 --chown=dynamo:0 tests/__init__.py /workspace/tests/__init__.py From 4e85bf656692dd7cd8fca9f78ecca9be3d3a0ccd Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 8 Sep 2026 19:36:57 -0700 Subject: [PATCH 11/12] fix(test): skip venv chmod in distroless images Signed-off-by: jain-ria --- container/Dockerfile.test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/container/Dockerfile.test b/container/Dockerfile.test index 79f5bcd28c40..7fb9ca24573e 100644 --- a/container/Dockerfile.test +++ b/container/Dockerfile.test @@ -37,9 +37,11 @@ RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/ # the backend. GitHub's ARC runner executes this image as an arbitrary uid, so # make only the test image's venv package directory writable by that uid. The # shipped runtime image remains read-only. -RUN if [ -n "${VIRTUAL_ENV:-}" ]; then \ +RUN if [ -n "${VIRTUAL_ENV:-}" ] && command -v chmod >/dev/null; then \ SITE_PACKAGES="$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" && \ chmod o+w "${SITE_PACKAGES}"; \ + else \ + echo "skipped writable test site-packages (no venv or chmod)"; \ fi # Copy pytest configuration (markers, filterwarnings, asyncio_mode, etc.) From 3696c92d43c18fdc66e409b669b9aa238f63dfbe Mon Sep 17 00:00:00 2001 From: jain-ria Date: Tue, 8 Sep 2026 20:49:32 -0700 Subject: [PATCH 12/12] test: isolate optional runtime package installs Signed-off-by: jain-ria --- container/Dockerfile.test | 11 ---- tests/serve/common.py | 69 +++++++++++++++----- tests/serve/test_common_helpers.py | 101 +++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 29 deletions(-) create mode 100644 tests/serve/test_common_helpers.py diff --git a/container/Dockerfile.test b/container/Dockerfile.test index 7fb9ca24573e..bf71afcd34ff 100644 --- a/container/Dockerfile.test +++ b/container/Dockerfile.test @@ -33,17 +33,6 @@ RUN --mount=type=bind,source=./container/deps/requirements.test.txt,target=/tmp/ --requirement /tmp/requirements.test.txt; \ fi -# Some serve tests add an optional media decoder immediately before launching -# the backend. GitHub's ARC runner executes this image as an arbitrary uid, so -# make only the test image's venv package directory writable by that uid. The -# shipped runtime image remains read-only. -RUN if [ -n "${VIRTUAL_ENV:-}" ] && command -v chmod >/dev/null; then \ - SITE_PACKAGES="$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')" && \ - chmod o+w "${SITE_PACKAGES}"; \ - else \ - echo "skipped writable test site-packages (no venv or chmod)"; \ - fi - # Copy pytest configuration (markers, filterwarnings, asyncio_mode, etc.) COPY --chmod=664 --chown=dynamo:0 pyproject.toml /workspace/pyproject.toml COPY --chmod=664 --chown=dynamo:0 tests/__init__.py /workspace/tests/__init__.py diff --git a/tests/serve/common.py b/tests/serve/common.py index 602f324801e1..e6e101fe055d 100644 --- a/tests/serve/common.py +++ b/tests/serve/common.py @@ -6,8 +6,10 @@ import dataclasses import logging import os +import shutil import subprocess import sys +import tempfile import time from collections.abc import Callable, Mapping from contextlib import contextmanager @@ -307,32 +309,63 @@ def managed_serve_deployment( # is retained without the shipped image carrying it. No-op when the key is unset. TEST_ONLY_PIP_ENV_KEY = "DYN_TEST_ONLY_PIP_INSTALL" -# Session-level guard so the same package set is installed at most once even +# Session-level cache so the same package set is installed at most once even # though every parametrized deployment (and each retry) calls the installer. -_test_only_pip_done: set[str] = set() +_test_only_pip_targets: dict[str, str] = {} -def _install_test_only_packages(config: EngineConfig) -> None: +def _install_test_only_packages( + config: EngineConfig, extra_env: Optional[Dict[str, str]] = None +) -> dict[str, str]: """Install any test-only pip packages a config requested via its env. - Runs inside the same runtime container/interpreter the server subprocess - inherits, so the worker can import the freshly installed module. + Install into a process-isolated temporary directory rather than the runtime + interpreter's site-packages. CI may run the image as an arbitrary uid, and + some framework venvs are intentionally read-only. The returned environment + exposes the directory only to subprocesses launched for this deployment. """ + launch_env = dict(extra_env or {}) spec = config.env.get(TEST_ONLY_PIP_ENV_KEY, "").strip() - if not spec or spec in _test_only_pip_done: - return - packages = spec.split() - logging.getLogger(__name__).info( - "Installing test-only package(s) into runtime container: %s", - " ".join(packages), + if not spec: + return launch_env + + target = _test_only_pip_targets.get(spec) + if target is None: + packages = spec.split() + target = tempfile.mkdtemp(prefix="dynamo-test-pip-") + logging.getLogger(__name__).info( + "Installing test-only package(s) into %s: %s", + target, + " ".join(packages), + ) + try: + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--target", + target, + "--no-deps", + *packages, + ], + check=True, + ) + except Exception: + shutil.rmtree(target, ignore_errors=True) + raise + _test_only_pip_targets[spec] = target + + inherited_pythonpath = launch_env.get( + "PYTHONPATH", config.env.get("PYTHONPATH", os.environ.get("PYTHONPATH", "")) ) - # --break-system-packages: runtime images use an externally-managed system - # python (PEP 668); this is the ephemeral test container, not a shipped image. - subprocess.run( - [sys.executable, "-m", "pip", "install", "--break-system-packages", *packages], - check=True, + launch_env["PYTHONPATH"] = ( + target + if not inherited_pythonpath + else os.pathsep.join((target, inherited_pythonpath)) ) - _test_only_pip_done.add(spec) + return launch_env def run_serve_deployment( @@ -363,7 +396,7 @@ def run_serve_deployment( # Install any decoder a codec-stripped image needs for this test, before the # server launches, so the worker can import it. No-op unless the config opts in. - _install_test_only_packages(config) + extra_env = _install_test_only_packages(config, extra_env) prep = _prepare_deployment(config, request, ports=ports, extra_env=extra_env) config = prep.config diff --git a/tests/serve/test_common_helpers.py b/tests/serve/test_common_helpers.py new file mode 100644 index 000000000000..7d5ed7ec9ac0 --- /dev/null +++ b/tests/serve/test_common_helpers.py @@ -0,0 +1,101 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest + +from tests.serve import common +from tests.utils.engine_process import EngineConfig + +pytestmark = [pytest.mark.pre_merge, pytest.mark.unit, pytest.mark.gpu_0] + + +def _config(*, spec: str = "") -> EngineConfig: + env = {common.TEST_ONLY_PIP_ENV_KEY: spec} if spec else {} + return EngineConfig( + name="test", + directory=".", + marks=[], + request_payloads=[], + model="test", + command=["true"], + env=env, + ) + + +@pytest.fixture(autouse=True) +def clear_test_only_pip_targets(): + common._test_only_pip_targets.clear() + yield + common._test_only_pip_targets.clear() + + +def test_install_test_only_packages_uses_isolated_target(monkeypatch, tmp_path): + target = tmp_path / "packages" + calls = [] + monkeypatch.setattr(common.tempfile, "mkdtemp", lambda **_kwargs: str(target)) + monkeypatch.setattr( + common.subprocess, + "run", + lambda cmd, **kwargs: calls.append((cmd, kwargs)), + ) + + env = common._install_test_only_packages( + _config(spec="decord2>=3.4.0,<4"), {"PYTHONPATH": "/existing"} + ) + + assert calls == [ + ( + [ + common.sys.executable, + "-m", + "pip", + "install", + "--target", + str(target), + "--no-deps", + "decord2>=3.4.0,<4", + ], + {"check": True}, + ) + ] + assert env["PYTHONPATH"] == f"{target}{common.os.pathsep}/existing" + + +def test_install_test_only_packages_reuses_successful_install(monkeypatch, tmp_path): + target = tmp_path / "packages" + calls = [] + monkeypatch.setattr(common.tempfile, "mkdtemp", lambda **_kwargs: str(target)) + monkeypatch.setattr( + common.subprocess, + "run", + lambda cmd, **kwargs: calls.append((cmd, kwargs)), + ) + config = _config(spec="decord2>=3.4.0,<4") + + first = common._install_test_only_packages(config) + second = common._install_test_only_packages(config) + + assert len(calls) == 1 + assert first["PYTHONPATH"] == str(target) + assert second["PYTHONPATH"] == str(target) + + +def test_install_test_only_packages_removes_failed_target(monkeypatch, tmp_path): + target = tmp_path / "packages" + target.mkdir() + marker = target / "partial-wheel" + marker.touch() + monkeypatch.setattr(common.tempfile, "mkdtemp", lambda **_kwargs: str(target)) + + def fail_install(*_args, **_kwargs): + raise RuntimeError("pip failed") + + monkeypatch.setattr(common.subprocess, "run", fail_install) + + with pytest.raises(RuntimeError, match="pip failed"): + common._install_test_only_packages(_config(spec="decord2>=3.4.0,<4")) + + assert not Path(target).exists() + assert not common._test_only_pip_targets