chore(sglang): bump to 0.5.17

.github/actions/setup-dynamo-operator/action.yml

@@ -470,6 +470,29 @@ runs:
       run: |
         echo "${DOCKERHUB_PASS}" | helm registry login registry-1.docker.io -u "${DOCKERHUB_USER}" --password-stdin
 
+    # The operator refuses to start when checkpoint is enabled unless the
+    # PodSnapshot API exists. Snapshot-agent (which owns that CRD) is installed
+    # in a later job, so apply the CRDs first and let that later job adopt them.
+    - name: Install snapshot CRDs for checkpoint
+      if: inputs.checkpoint_enabled == 'true'
+      shell: bash
+      env:
+        # Keep in lockstep with .github/actions/setup-snapshot-agent/action.yml.
+        SNAPSHOT_CHART_VERSION: 0.1.0-alpha.1
+      run: |
+        echo "::group::Install snapshot CRDs before operator start"
+        set -euo pipefail
+        VKUBECONFIG="${{ github.workspace }}/.kubeconfig-vcluster"
+        helm show crds oci://ghcr.io/ai-dynamo/snapshot/snapshot \
+          --version "${SNAPSHOT_CHART_VERSION}" \
+          | kubectl --kubeconfig="${VKUBECONFIG}" \
+              apply --server-side --force-conflicts --field-manager=ci-setup -f -
+        kubectl --kubeconfig="${VKUBECONFIG}" wait --for=condition=established \
+          crd/podsnapshots.nvidia.com \
+          crd/podsnapshotcontents.nvidia.com \
+          --timeout=120s
+        echo "::endgroup::"
+
     - name: Install Dynamo platform via Helm
       shell: bash
       env:

components/src/dynamo/sglang/CLAUDE.md

@@ -20,7 +20,7 @@ support the current version plus 1 version back (N and N-1). The pattern:
    enough surface area to cover what Dynamo actually calls.
 4. Each fallback branch in `_compat.py` MUST have a comment noting which SGLang
    version it supports and when it can be removed, e.g.:
-   `# Fallback for sglang <= 0.5.12. Remove when min supported version is 0.5.14+`
+   `# Fallback for sglang <= 0.5.16. Remove when min supported version is 0.5.18+`
 5. When a new SGLang version is released and the old N-1 falls outside the support
    window, delete the corresponding fallback branches and polyfills from `_compat.py`.
    If `_compat.py` becomes trivial re-exports, inline the imports and delete the file.
@@ -70,6 +70,11 @@ Worker dispatch (main.py:60-132):
    have `max_running_requests`, `dllm_algorithm_config`, or other LLM-specific fields.
    Use `getattr()` when accessing fields that may not exist on the stub.
 
+SGLang 0.5.17 makes a resolved `ServerArgs` unconditionally read-only. Apply Dynamo's
+post-resolution startup overrides through `_compat.override_server_args()`; control-plane
+updates after engine creation should use the tokenizer manager's update API instead of
+assigning fields on `server_args`.
+
 **DynamoConfig** combines `DynamoRuntimeConfig` (common flags like `--namespace`,
 `--output-modalities`, `--media-output-fs-url`) with `DynamoSGLangConfig` (sglang-specific
 flags like `--enable-multimodal`, `--dedicated-mm-encoder`, `--embedding-worker`).

components/src/dynamo/sglang/_compat.py

@@ -38,35 +38,10 @@ def _warn_require_reasoning_unsupported() -> None:
     )
 
 
-# ---------------------------------------------------------------------------
-# Top-level sglang exports: Engine, ServerArgs
-#
-# Some SGLang dev builds (including 0.5.x snapshots) do not re-export these
-# from sglang/__init__.py, while Dynamo historically uses `import sglang as sgl`
-# followed by `sgl.Engine(...)` throughout this backend.
-# ---------------------------------------------------------------------------
-def ensure_sglang_top_level_exports() -> None:
-    """Restore top-level SGLang exports omitted by some install flavors."""
-    import sglang as sgl
-
-    if not hasattr(sgl, "Engine"):
-        from sglang.srt.entrypoints.engine import Engine
-
-        sgl.Engine = Engine
-
-    if not hasattr(sgl, "ServerArgs"):
-        from sglang.srt.server_args import ServerArgs
-
-        sgl.ServerArgs = ServerArgs
-
-
-ensure_sglang_top_level_exports()
-
-
 def ensure_sglang_tensor_image_size() -> None:
     """Allow SGLang's image-token resolver to handle decoded image tensors.
 
-    SGLang 0.5.13 through 0.5.16 assume every decoded image exposes the PIL
+    SGLang 0.5.13 through 0.5.17 assume every decoded image exposes the PIL
     ``height``/``width`` attributes. Its CUDA JPEG decoder instead returns a
     CHW tensor, causing multimodal requests to fall back to retokenization.
 
@@ -106,6 +81,27 @@ def resolve_image_token_counts(self: Any, images: list[Any]) -> list[int]:
     BaseMultimodalProcessor.resolve_image_token_counts = resolve_image_token_counts
 
 
+def override_server_args(server_args: Any, source: str, **fields: Any) -> None:
+    """Apply a post-resolution SGLang configuration update.
+
+    SGLang 0.5.17 makes ``ServerArgs`` unconditionally read-only after
+    resolution. Both supported CUDA releases expose ``ServerArgs.override`` as
+    the audited mutation API, so Dynamo must use it instead of assigning fields.
+    The separately pinned XPU image still uses SGLang 0.5.11, which predates
+    that API; preserve its legacy assignment behavior until its engine pin is
+    upgraded.
+    """
+    override = getattr(server_args, "override", None)
+    if callable(override):
+        override(source, **fields)
+        return
+
+    # XPU compatibility for SGLang 0.5.11. Remove when the XPU SGLang pin is
+    # upgraded to 0.5.16+.
+    for name, value in fields.items():
+        setattr(server_args, name, value)
+
+
 @lru_cache(maxsize=32)
 def _get_async_generate_supported_kwarg_names(
     async_generate: Any,
@@ -138,10 +134,12 @@ def filter_supported_async_generate_kwargs(
 ) -> dict[str, Any]:
     """Return only async_generate kwargs accepted by this SGLang engine.
 
-    SGLang occasionally adds optional Engine.async_generate kwargs before every
-    supported install flavor has them. Keep the compatibility boundary narrow:
-    callers decide which kwargs are optional, and this helper only drops those
-    optional kwargs when the installed engine cannot accept them.
+    Both supported CUDA releases accept Dynamo's optional kwargs. The separately
+    pinned XPU image still uses SGLang 0.5.11, which predates ``mm_hashes`` and
+    ``require_reasoning``. Keep the compatibility boundary narrow: callers
+    decide which kwargs are optional, and this helper only drops those optional
+    kwargs when the installed engine cannot accept them. Remove this filtering
+    when the XPU SGLang pin is upgraded to 0.5.16+.
     """
     async_generate = engine.async_generate
     signature_source = getattr(async_generate, "__func__", async_generate)
@@ -175,7 +173,7 @@ def require_reasoning_kwargs(engine: Any, request: Mapping[str, Any]) -> dict[st
 
 __all__ = [
     "ensure_sglang_tensor_image_size",
-    "ensure_sglang_top_level_exports",
     "filter_supported_async_generate_kwargs",
+    "override_server_args",
     "require_reasoning_kwargs",
 ]

components/src/dynamo/sglang/init_llm.py

@@ -14,6 +14,7 @@
 from dynamo.common.utils.endpoint_types import parse_endpoint_types
 from dynamo.llm import ModelInput, ModelType, WorkerType
 from dynamo.runtime import DistributedRuntime
+from dynamo.sglang._compat import override_server_args
 from dynamo.sglang.args import Config
 from dynamo.sglang.health_check import (
     SglangDisaggHealthCheckPayload,
@@ -71,7 +72,11 @@ async def init_decode(
                 "created before the endpoint existed, so its FPM publisher bound "
                 "a different IPC path than the relay would subscribe to."
             )
-            server_args.enable_forward_pass_metrics = False
+            override_server_args(
+                server_args,
+                "dynamo.snapshot",
+                enable_forward_pass_metrics=False,
+            )
     else:
         set_forward_pass_metrics_worker_id(server_args, generate_endpoint)
         start_time = time.time()
@@ -227,7 +232,11 @@ async def init_prefill(
                 "created before the endpoint existed, so its FPM publisher bound "
                 "a different IPC path than the relay would subscribe to."
             )
-            server_args.enable_forward_pass_metrics = False
+            override_server_args(
+                server_args,
+                "dynamo.snapshot",
+                enable_forward_pass_metrics=False,
+            )
     else:
         set_forward_pass_metrics_worker_id(server_args, generate_endpoint)
         start_time = time.time()

components/src/dynamo/sglang/init_multimodal.py

@@ -253,7 +253,7 @@ async def init_multimodal_prefill_worker(
         logging.error(f"Failed to serve endpoints: {e}")
         raise
     finally:
-        handler.cleanup()
+        await handler.cleanup_async()
         if run_deferred_handlers is not None:
             logging.info("Running deferred handlers")
             await run_deferred_handlers()

components/src/dynamo/sglang/main.py

@@ -15,6 +15,7 @@
 )
 from dynamo.common.utils.runtime import create_runtime
 from dynamo.runtime.logging import configure_dynamo_logging
+from dynamo.sglang._compat import override_server_args
 from dynamo.sglang.args import parse_args
 from dynamo.sglang.init_diffusion import (
     init_image_diffusion,
@@ -44,7 +45,11 @@ async def worker(argv: list[str] | None = None):
     if config.server_args.load_format == "gms":
         from gpu_memory_service.integrations.sglang import setup_gms
 
-        config.server_args.load_format = setup_gms(config.server_args)
+        override_server_args(
+            config.server_args,
+            "dynamo.gms",
+            load_format=setup_gms(config.server_args),
+        )
 
     # Snapshot mode: engine must be created before runtime so CRIU captures no
     # NATS/etcd connections.

components/src/dynamo/sglang/publisher.py

@@ -24,6 +24,7 @@
 )
 from dynamo.llm import KvEventPublisher, WorkerMetricsPublisher
 from dynamo.runtime import Endpoint
+from dynamo.sglang._compat import override_server_args
 from dynamo.sglang._disagg import SGLANG_WORKER_GROUP_ID_KEY, get_sglang_worker_group_id
 from dynamo.sglang.args import Config
 from dynamo.sglang.capacity import (
@@ -49,9 +50,13 @@ def set_forward_pass_metrics_worker_id(
 
     import tempfile
 
-    server_args.forward_pass_metrics_worker_id = str(generate_endpoint.connection_id())
     ipc_path = tempfile.NamedTemporaryFile(delete=False).name
-    server_args.forward_pass_metrics_ipc_name = f"ipc://{ipc_path}"
+    override_server_args(
+        server_args,
+        "dynamo.forward_pass_metrics",
+        forward_pass_metrics_worker_id=str(generate_endpoint.connection_id()),
+        forward_pass_metrics_ipc_name=f"ipc://{ipc_path}",
+    )
 
 
 async def _resolve_multinode_leader_worker_id(

components/src/dynamo/sglang/request_handlers/handler_base.py

@@ -871,7 +871,9 @@ async def update_weight_version(self, body: dict) -> dict:
         if req.abort_all_requests:
             self.engine.tokenizer_manager.abort_request(abort_all=True)
 
-        self.engine.tokenizer_manager.server_args.weight_version = req.new_version
+        self.engine.tokenizer_manager._update_weight_version_if_provided(
+            req.new_version
+        )
         return {
             "success": True,
             "message": f"Weight version updated to {req.new_version}",