From a888122507f944f3a2b41a1fc36a5fb103357447 Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Tue, 8 Sep 2026 05:54:36 +0000 Subject: [PATCH 01/11] fix(sglang): fail fast when --elastic-ep-backend mooncake is unusable SGLang builds its elastic-EP process groups from the mooncake transfer engine's torch ProcessGroup extension, registering "mooncake" (device) and "mooncake-cpu" (CPU) as torch.distributed backends. Every collective those groups run goes through that extension, including the all-gather that --enable-dp-attention issues to sync MLP batch metadata on the first forward pass. The extension is compiled per exact torch release (pg___) and was renamed mooncake.ep -> mooncake.pg between SGLang v0.5.16 and v0.5.18, so an image can carry mooncake and still have no usable backend. Today that surfaces after the model has loaded, from inside upstream engine code, with no mention of the flag that caused it. Check the same preconditions while Dynamo is still assembling ServerArgs and raise a ValueError naming the requested flag, the extension import failure, the installed mooncake distributions, the running torch version, and the extensions the wheel actually ships. The check is silent for every other --elastic-ep-backend value and when the flag is unset. The preflight lives in its own module that imports no sglang symbol and imports torch only inside its probe helpers, so it stays unit-testable in an image where the engine is not importable. The root conftest's engine-seeding guard is broadened from ImportError to Exception for the same reason: an editable install whose source tree is unreadable raises PermissionError and was aborting collection of the entire suite. Also document the mooncake ProcessGroup requirement in the elastic-EP fault-tolerance deployment template. Signed-off-by: svc-glamr@nvidia.com --- components/src/dynamo/sglang/args.py | 6 + .../src/dynamo/sglang/elastic_ep_preflight.py | 198 ++++++++++++++++++ .../sglang/tests/test_elastic_ep_preflight.py | 136 ++++++++++++ conftest.py | 6 +- .../templates/sglang/sglang_elastic_ep.yaml | 15 ++ 5 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 components/src/dynamo/sglang/elastic_ep_preflight.py create mode 100644 components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py diff --git a/components/src/dynamo/sglang/args.py b/components/src/dynamo/sglang/args.py index a3fe2882786e..7b85b12317cb 100644 --- a/components/src/dynamo/sglang/args.py +++ b/components/src/dynamo/sglang/args.py @@ -40,6 +40,7 @@ resolved_server_args, ) from dynamo.sglang.backend_args import DynamoSGLangArgGroup, DynamoSGLangConfig +from dynamo.sglang.elastic_ep_preflight import check_elastic_ep_backend configure_dynamo_logging() PREFILL_DECODE_DISAGGREGATION_MODE = "pd" @@ -628,6 +629,11 @@ async def parse_args(args: list[str]) -> Config: if get_sglang_model_config(server_args).is_multimodal: ensure_sglang_tensor_image_size() + check_elastic_ep_backend( + getattr(server_args, "elastic_ep_backend", None), + getattr(server_args, "enable_dp_attention", False), + ) + if getattr(server_args, "schedule_low_priority_values_first", False): raise ValueError( "--schedule-low-priority-values-first is not supported in Dynamo's " diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py new file mode 100644 index 000000000000..070d32b123a9 --- /dev/null +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -0,0 +1,198 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Startup check for SGLang's ``--elastic-ep-backend mooncake``. + +SGLang builds its elastic-EP process groups from the mooncake transfer +engine's torch ``ProcessGroup`` extension: ``backend="mooncake"`` for the +device group and ``backend="mooncake-cpu"`` for the CPU group. That import +and that group construction happen deep inside engine startup, well after +the model is loaded, so a worker whose image cannot serve the requested +backend only finds out late and reports it from inside upstream engine code +that the deployer did not write. Checking the same preconditions while +Dynamo is still assembling ``ServerArgs`` turns that into an argument error +naming the flag that caused it. + +This module deliberately imports no ``sglang`` symbol, and imports ``torch`` +only inside the probe helpers. That keeps it usable -- and unit-testable -- +in an image where the engine is not importable. +""" + +from __future__ import annotations + +import importlib +import importlib.util +import os +from typing import Dict, List, Optional, Tuple + +MOONCAKE_BACKEND = "mooncake" + +# The torch backend name SGLang registers the elastic-EP *device* group under +# (sglang/srt/distributed/parallel_state.py). The CPU group's "mooncake-cpu" is +# reported for context but not required: an engine/wheel pairing that only needs +# the device group must not be blocked here. +_MOONCAKE_DEVICE_BACKEND = "mooncake" + +# mooncake renamed this extension ``mooncake.ep`` -> ``mooncake.pg``; SGLang +# v0.5.16 imports the old name and v0.5.18 the new one. Either resolving is +# enough, so try both rather than pinning the engine version this Dynamo build +# happens to sit next to. +_PROCESS_GROUP_MODULES = ("mooncake.pg", "mooncake.ep") + +# Both distributions ship the same extension; only one is normally installed. +_MOONCAKE_DISTRIBUTIONS = ( + "mooncake-transfer-engine-cuda13", + "mooncake-transfer-engine", +) + +# Compiled ProcessGroup extensions are named for the exact torch release they +# were built against, e.g. ``pg_2_11_0`` / ``ep_2_9_1``. +_EXTENSION_PREFIXES = ("pg_", "ep_") + + +def _import_process_group_extension() -> Optional[str]: + """Return ``None`` when the extension imports, else the collected failures. + + Importing it is what SGLang itself does later; doing it here surfaces the + torch-version skew (``mooncake.pg`` raises ``ImportError`` when no + ``pg_`` module matches) before any GPU work has started. + """ + failures: List[str] = [] + for module_name in _PROCESS_GROUP_MODULES: + try: + importlib.import_module(module_name) + return None + except Exception as exc: # noqa: BLE001 - any failure means "unusable" + failures.append(f"{module_name}: {type(exc).__name__}: {exc}") + return "; ".join(failures) if failures else "no attempt was made" + + +def _installed_mooncake_versions() -> Dict[str, str]: + """Map each installed mooncake distribution to its version.""" + from importlib.metadata import PackageNotFoundError + from importlib.metadata import version as _distribution_version + + found: Dict[str, str] = {} + for distribution in _MOONCAKE_DISTRIBUTIONS: + try: + found[distribution] = _distribution_version(distribution) + except PackageNotFoundError: + continue + return found + + +def _available_extension_modules() -> List[str]: + """Names of the compiled ProcessGroup extensions the wheel actually ships. + + Resolved from the package directory rather than by import, so this stays + cheap and side-effect free when the extension is the thing that is broken. + """ + try: + spec = importlib.util.find_spec("mooncake") + except Exception: # noqa: BLE001 - a broken package must not mask the real error + return [] + if spec is None or not spec.submodule_search_locations: + return [] + + modules = set() + for location in spec.submodule_search_locations: + try: + entries = os.listdir(location) + except OSError: + continue + for entry in entries: + if not entry.startswith(_EXTENSION_PREFIXES): + continue + modules.add(entry.split(".", 1)[0]) + return sorted(modules) + + +def _registered_torch_backends() -> Dict[str, Tuple[str, ...]]: + """torch's backend -> supported-device map, or empty when torch is absent.""" + try: + import torch.distributed as torch_distributed + + return dict(torch_distributed.Backend.backend_capability) + except Exception: # noqa: BLE001 - reported as "nothing registered" + return {} + + +def _torch_version() -> str: + try: + import torch + + return str(torch.__version__) + except Exception: # noqa: BLE001 + return "not installed" + + +def _format_versions(versions: Dict[str, str]) -> str: + if not versions: + return "none installed" + return ", ".join(f"{name} {value}" for name, value in sorted(versions.items())) + + +def _build_diagnostic( + import_failure: Optional[str], + registered_backends: Dict[str, Tuple[str, ...]], + enable_dp_attention: bool, +) -> str: + mooncake_backends = { + name: devices + for name, devices in registered_backends.items() + if MOONCAKE_BACKEND in name + } + lines = [ + "--elastic-ep-backend mooncake was requested, but the mooncake torch " + "ProcessGroup backend is not usable in this image. SGLang builds its " + "elastic-EP process groups from that backend after the model is " + "loaded, so leaving this unchecked fails the worker late and from " + "inside engine code.", + f" mooncake ProcessGroup extension: {import_failure or 'imported, but registered no usable backend'}", + f" installed mooncake distributions: {_format_versions(_installed_mooncake_versions())}", + f" torch: {_torch_version()}", + f" ProcessGroup extensions this mooncake ships: {', '.join(_available_extension_modules()) or 'none found'}", + f" mooncake backends registered with torch.distributed: {mooncake_backends or 'none'}", + ] + if enable_dp_attention: + lines.append( + " --enable-dp-attention is also set: DP attention synchronizes its " + "MLP batch metadata with an all-gather over this group on the very " + "first forward pass, so this combination reaches the backend " + "immediately at startup." + ) + lines.append( + "Install a mooncake-transfer-engine build whose ProcessGroup extension " + "matches the running torch, or drop --elastic-ep-backend." + ) + return "\n".join(lines) + + +def check_elastic_ep_backend( + elastic_ep_backend: Optional[str], + enable_dp_attention: bool = False, +) -> None: + """Reject ``--elastic-ep-backend mooncake`` when the image cannot serve it. + + Silent for every other configuration, including an unset backend: this must + never turn into an unconditional startup failure for workers that asked for + nothing of the sort. + + Raises: + ValueError: The mooncake backend was requested but its torch + ProcessGroup extension does not import, or imports without + registering the device backend SGLang asks torch for. + """ + if not elastic_ep_backend: + return + if str(elastic_ep_backend).strip().lower() != MOONCAKE_BACKEND: + return + + import_failure = _import_process_group_extension() + registered_backends = {} if import_failure else _registered_torch_backends() + if import_failure is None and _MOONCAKE_DEVICE_BACKEND in registered_backends: + return + + raise ValueError( + _build_diagnostic(import_failure, registered_backends, enable_dp_attention) + ) diff --git a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py new file mode 100644 index 000000000000..ebeeef9e88e9 --- /dev/null +++ b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py @@ -0,0 +1,136 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Behaviour of the ``--elastic-ep-backend mooncake`` startup check. + +Imports neither ``sglang`` nor any ``dynamo.sglang`` module that imports it, +so these run in an image where the engine is not installed or not importable +-- which is the only reason the check lives in its own module rather than in +``args.py``. The environment probes are the seams; the decision the check +makes over their results is what is exercised here. +""" + +import pytest + +from dynamo.sglang import elastic_ep_preflight +from dynamo.sglang.elastic_ep_preflight import check_elastic_ep_backend + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + +_IMPORT_FAILURE = ( + "mooncake.pg: ImportError: Mooncake PG was not built against torch==2.11.0; " + "mooncake.ep: ModuleNotFoundError: No module named 'mooncake.ep'" +) + + +def _simulate_environment( + monkeypatch, + *, + import_failure, + registered_backends, + mooncake_versions, + extension_modules, + torch_version="2.11.0+cu130", +): + """Pin every probe so the check runs against a known image shape.""" + monkeypatch.setattr( + elastic_ep_preflight, + "_import_process_group_extension", + lambda: import_failure, + ) + monkeypatch.setattr( + elastic_ep_preflight, + "_registered_torch_backends", + lambda: dict(registered_backends), + ) + monkeypatch.setattr( + elastic_ep_preflight, + "_installed_mooncake_versions", + lambda: dict(mooncake_versions), + ) + monkeypatch.setattr( + elastic_ep_preflight, + "_available_extension_modules", + lambda: list(extension_modules), + ) + monkeypatch.setattr(elastic_ep_preflight, "_torch_version", lambda: torch_version) + + +def _simulate_broken_mooncake(monkeypatch, **overrides): + """An image whose mooncake ProcessGroup extension does not load.""" + kwargs = { + "import_failure": _IMPORT_FAILURE, + "registered_backends": {"gloo": ("cpu",), "nccl": ("cuda",)}, + "mooncake_versions": {"mooncake-transfer-engine-cuda13": "0.3.11.post1"}, + "extension_modules": ["pg_2_9_1", "pg_2_10_0"], + } + kwargs.update(overrides) + _simulate_environment(monkeypatch, **kwargs) + + +def _simulate_healthy_mooncake(monkeypatch): + """An image whose mooncake ProcessGroup extension loads and registers.""" + _simulate_environment( + monkeypatch, + import_failure=None, + registered_backends={ + "gloo": ("cpu",), + "nccl": ("cuda",), + "mooncake": ("cuda",), + "mooncake-cpu": ("cpu",), + }, + mooncake_versions={"mooncake-transfer-engine-cuda13": "0.3.11.post1"}, + extension_modules=["pg_2_11_0"], + ) + + +def test_rejects_mooncake_backend_the_image_cannot_serve(monkeypatch): + """The worker fails at argument parsing, naming what the operator needs.""" + _simulate_broken_mooncake(monkeypatch) + + with pytest.raises(ValueError) as excinfo: + check_elastic_ep_backend("mooncake", True) + + message = str(excinfo.value) + assert "--elastic-ep-backend" in message + # The three facts that separate a bad wheel from a torch-version skew, so + # the failure is diagnosable without reproducing the crash. + assert "0.3.11.post1" in message + assert "2.11.0+cu130" in message + assert "pg_2_9_1" in message + # --enable-dp-attention is what drives the collective on the first forward + # pass, so it has to be called out when it is part of the configuration. + assert "--enable-dp-attention" in message + + +def test_reports_missing_mooncake_install_explicitly(monkeypatch): + """No mooncake at all reads as an absence, not a blank version field.""" + _simulate_broken_mooncake(monkeypatch, mooncake_versions={}, extension_modules=[]) + + with pytest.raises(ValueError) as excinfo: + check_elastic_ep_backend("mooncake") + + assert "none installed" in str(excinfo.value) + + +@pytest.mark.parametrize("requested_backend", [None, "", "nccl", "nvshmem"]) +def test_ignores_backends_other_than_mooncake(monkeypatch, requested_backend): + """Negative control: the same broken image is fine for everyone else. + + Guards against the check becoming an unconditional startup failure for + workers that never asked for the mooncake transport. + """ + _simulate_broken_mooncake(monkeypatch) + + check_elastic_ep_backend(requested_backend, True) + + +def test_accepts_mooncake_when_the_image_can_serve_it(monkeypatch): + """Negative control: a working image is not blocked.""" + _simulate_healthy_mooncake(monkeypatch) + + check_elastic_ep_backend("mooncake", True) diff --git a/conftest.py b/conftest.py index 590217e737a3..bd199649e78b 100644 --- a/conftest.py +++ b/conftest.py @@ -35,10 +35,14 @@ _REPO_ROOT = Path(__file__).resolve().parent # Seed sys.modules with the venv copies before pytest collection runs. +# Best-effort by construction: an engine that is present but not importable -- +# an editable install whose source tree is unreadable raises PermissionError, +# not ImportError -- must not abort collection of the entire suite, so catch +# everything the import can raise rather than ImportError alone. for _name in ("vllm", "sglang"): try: importlib.import_module(_name) - except ImportError: + except Exception: pass # Suppress ImportPathMismatchError when pytest later loads dynamo.vllm diff --git a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml index 4f7a5c281be4..e426d012ed0e 100644 --- a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml +++ b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml @@ -29,6 +29,21 @@ # 4 idle warm-standby GPUs for the joining group (scale 4 -> 8). # * RDMA / InfiniBand for the mooncake weight-transfer backend. Set # --mooncake-ib-device to the node's IB device list before applying. +# * A mooncake-transfer-engine build that ships the torch ProcessGroup +# extension. --elastic-ep-backend mooncake is not just the weight transport: +# SGLang registers "mooncake" (device) and "mooncake-cpu" (CPU) as +# torch.distributed backends and builds the elastic-EP process groups from +# them, so every collective those groups run -- including the all-gather +# that --enable-dp-attention issues to sync MLP batch metadata -- goes +# through it. Two ways the image can be wrong: +# - The extension is compiled per torch release (pg___, +# e.g. pg_2_11_0). A wheel with no module matching the image's exact +# torch version has no usable backend. +# - mooncake renamed the module mooncake.ep -> mooncake.pg; SGLang +# v0.5.16 imports the old name and v0.5.18 the new one, so the engine +# and the wheel have to agree on which one exists. +# The Dynamo SGLang worker checks both while parsing arguments and refuses +# to start with a diagnostic rather than failing after the model loads. # * DYN_SYSTEM_ENABLED=true so the leader exposes the :9090 system server that # serves /engine/control/*. # From c88726f480c09b8956a23468d071a76b67b99052 Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Tue, 8 Sep 2026 06:34:52 +0000 Subject: [PATCH 02/11] fix(sglang): narrow the elastic-EP preflight claim and fail open on torch's registry Review follow-up on the mooncake elastic-EP startup check. The deployment template said the worker checks both documented failure modes. It checks one. `_import_process_group_extension()` accepts either `mooncake.pg` or `mooncake.ep` resolving, so it catches a wheel with no extension for the running torch, but a wheel that ships only the module name the installed engine does not import still passes and fails after model load. Say that instead of claiming both. `_registered_torch_backends()` now returns `None` when it cannot read `torch.distributed.Backend.backend_capability`, distinct from an empty registry, and the check skips only that assertion in that case. Reading a torch internal must not be able to refuse a working image; the extension import stays fatal. `conftest.py` prints one stderr line when an engine is installed but not importable, so a single unreadable install is named once instead of surfacing as a pile of unrelated-looking collection errors. An absent engine stays silent. Adds three tests: the fail-open registry case, and two that run the probe helpers themselves rather than the seams that stand in for them. Signed-off-by: svc-glamr@nvidia.com --- .../src/dynamo/sglang/elastic_ep_preflight.py | 29 +++++++--- .../sglang/tests/test_elastic_ep_preflight.py | 57 +++++++++++++++++++ conftest.py | 13 ++++- .../templates/sglang/sglang_elastic_ep.yaml | 8 ++- 4 files changed, 96 insertions(+), 11 deletions(-) diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py index 070d32b123a9..0cac2bc883aa 100644 --- a/components/src/dynamo/sglang/elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -107,14 +107,21 @@ def _available_extension_modules() -> List[str]: return sorted(modules) -def _registered_torch_backends() -> Dict[str, Tuple[str, ...]]: - """torch's backend -> supported-device map, or empty when torch is absent.""" +def _registered_torch_backends() -> Optional[Dict[str, Tuple[str, ...]]]: + """torch's backend -> supported-device map, or ``None`` when unreadable. + + ``Backend.backend_capability`` is a torch internal. Reporting "could not + read the registry" as "mooncake is not registered" would let a rename in + somebody else's private attribute refuse every worker that asked for this + transport, which is more fatal than the failure being guarded against. + ``None`` keeps the two apart so the caller can skip only this assertion. + """ try: import torch.distributed as torch_distributed return dict(torch_distributed.Backend.backend_capability) - except Exception: # noqa: BLE001 - reported as "nothing registered" - return {} + except Exception: # noqa: BLE001 - unreadable, not "nothing registered" + return None def _torch_version() -> str: @@ -189,9 +196,17 @@ def check_elastic_ep_backend( return import_failure = _import_process_group_extension() - registered_backends = {} if import_failure else _registered_torch_backends() - if import_failure is None and _MOONCAKE_DEVICE_BACKEND in registered_backends: - return + registered_backends: Dict[str, Tuple[str, ...]] = {} + if import_failure is None: + readable_backends = _registered_torch_backends() + # An unreadable registry fails open: the import above is what catches + # the reported failure, and refusing startup because torch moved a + # private attribute would block images that are perfectly fine. + if readable_backends is None: + return + if _MOONCAKE_DEVICE_BACKEND in readable_backends: + return + registered_backends = readable_backends raise ValueError( _build_diagnostic(import_failure, registered_backends, enable_dp_attention) diff --git a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py index ebeeef9e88e9..b09b856bdb49 100644 --- a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py @@ -10,6 +10,8 @@ makes over their results is what is exercised here. """ +import sys + import pytest from dynamo.sglang import elastic_ep_preflight @@ -134,3 +136,58 @@ def test_accepts_mooncake_when_the_image_can_serve_it(monkeypatch): _simulate_healthy_mooncake(monkeypatch) check_elastic_ep_backend("mooncake", True) + + +def test_unreadable_torch_backend_registry_does_not_block_startup(monkeypatch): + """A registry that cannot be read is not evidence that mooncake is absent. + + ``Backend.backend_capability`` is a torch internal. Reading it and failing + must not be worth more than the extension importing cleanly, or a torch-side + rename refuses every worker on this transport. + """ + _simulate_broken_mooncake(monkeypatch, import_failure=None) + monkeypatch.setattr( + elastic_ep_preflight, "_registered_torch_backends", lambda: None + ) + + check_elastic_ep_backend("mooncake", True) + + +def test_import_probe_reports_every_module_name_it_tried(monkeypatch): + """The probe itself, not a stand-in for it. + + The other cases replace the probes to pin the decision logic, which leaves + the probes themselves untested. This one runs the real + ``_import_process_group_extension`` against an import system that has no + mooncake in it, so the import, the exception capture, and the message + assembly are all the shipped code. Blocking at ``sys.meta_path`` rather + than by uninstalling makes it behave the same in an image that does have a + working wheel. + """ + + class _RefuseMooncake: + def find_spec(self, name, path=None, target=None): + if name == "mooncake" or name.startswith("mooncake."): + raise ImportError(f"blocked for this test: {name}") + return None + + monkeypatch.setattr(sys, "meta_path", [_RefuseMooncake(), *sys.meta_path]) + for cached in [ + name + for name in sys.modules + if name == "mooncake" or name.startswith("mooncake.") + ]: + monkeypatch.delitem(sys.modules, cached) + + failure = elastic_ep_preflight._import_process_group_extension() + + assert failure is not None + # Both names have to appear: which one an image ships is the first thing + # an operator needs to compare against the engine version. + assert "mooncake.pg" in failure + assert "mooncake.ep" in failure + + +def test_absent_mooncake_renders_as_an_explicit_absence(): + """No mooncake installed must read as words, not an empty field.""" + assert elastic_ep_preflight._format_versions({}) == "none installed" diff --git a/conftest.py b/conftest.py index bd199649e78b..de1e95b47551 100644 --- a/conftest.py +++ b/conftest.py @@ -42,8 +42,17 @@ for _name in ("vllm", "sglang"): try: importlib.import_module(_name) - except Exception: - pass + except Exception as _exc: # noqa: BLE001 - best-effort seeding, see above + # An engine that is simply absent is the normal case and stays quiet. + # Anything else means the install is present but broken, and every + # `import ` downstream will fail as an ordinary-looking + # collection error; one line here names the single root cause. + if not (isinstance(_exc, ModuleNotFoundError) and _exc.name == _name): + print( + f"conftest: {_name} is installed but could not be imported, so " + f"tests that import it will fail: {type(_exc).__name__}: {_exc}", + file=sys.stderr, + ) # Suppress ImportPathMismatchError when pytest later loads dynamo.vllm # under the bare name "vllm". diff --git a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml index e426d012ed0e..fc79d03f588c 100644 --- a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml +++ b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml @@ -42,8 +42,12 @@ # - mooncake renamed the module mooncake.ep -> mooncake.pg; SGLang # v0.5.16 imports the old name and v0.5.18 the new one, so the engine # and the wheel have to agree on which one exists. -# The Dynamo SGLang worker checks both while parsing arguments and refuses -# to start with a diagnostic rather than failing after the model loads. +# While parsing arguments the Dynamo SGLang worker tries both module names +# and refuses to start with a diagnostic if neither resolves, so the first +# case fails fast instead of after the model loads. It does NOT catch the +# second: either name resolving is accepted, so a wheel that ships only the +# name this engine does not import still passes the check and then fails +# inside the engine. Match the wheel to the engine version by hand. # * DYN_SYSTEM_ENABLED=true so the leader exposes the :9090 system server that # serves /engine/control/*. # From b3d4c70beee8d0def0f20f4f1313c2a3b270f75c Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Tue, 8 Sep 2026 07:18:06 +0000 Subject: [PATCH 03/11] chore: remove factory working notes Signed-off-by: svc-glamr@nvidia.com --- .../src/dynamo/sglang/elastic_ep_preflight.py | 13 ++++-------- conftest.py | 12 ++++------- .../templates/sglang/sglang_elastic_ep.yaml | 21 ++----------------- 3 files changed, 10 insertions(+), 36 deletions(-) diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py index 0cac2bc883aa..4fd1b21a9083 100644 --- a/components/src/dynamo/sglang/elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -28,15 +28,11 @@ MOONCAKE_BACKEND = "mooncake" # The torch backend name SGLang registers the elastic-EP *device* group under -# (sglang/srt/distributed/parallel_state.py). The CPU group's "mooncake-cpu" is -# reported for context but not required: an engine/wheel pairing that only needs -# the device group must not be blocked here. +# (sglang/srt/distributed/parallel_state.py). "mooncake-cpu" is not required. _MOONCAKE_DEVICE_BACKEND = "mooncake" # mooncake renamed this extension ``mooncake.ep`` -> ``mooncake.pg``; SGLang -# v0.5.16 imports the old name and v0.5.18 the new one. Either resolving is -# enough, so try both rather than pinning the engine version this Dynamo build -# happens to sit next to. +# v0.5.16 imports the old name and v0.5.18 the new one, so try both. _PROCESS_GROUP_MODULES = ("mooncake.pg", "mooncake.ep") # Both distributions ship the same extension; only one is normally installed. @@ -199,9 +195,8 @@ def check_elastic_ep_backend( registered_backends: Dict[str, Tuple[str, ...]] = {} if import_failure is None: readable_backends = _registered_torch_backends() - # An unreadable registry fails open: the import above is what catches - # the reported failure, and refusing startup because torch moved a - # private attribute would block images that are perfectly fine. + # An unreadable registry fails open: the import above already catches + # the reported failure, and torch could rename a private attribute. if readable_backends is None: return if _MOONCAKE_DEVICE_BACKEND in readable_backends: diff --git a/conftest.py b/conftest.py index de1e95b47551..747089941f8a 100644 --- a/conftest.py +++ b/conftest.py @@ -35,18 +35,14 @@ _REPO_ROOT = Path(__file__).resolve().parent # Seed sys.modules with the venv copies before pytest collection runs. -# Best-effort by construction: an engine that is present but not importable -- -# an editable install whose source tree is unreadable raises PermissionError, -# not ImportError -- must not abort collection of the entire suite, so catch -# everything the import can raise rather than ImportError alone. +# Best-effort: an engine that is present but not importable (an editable install +# with an unreadable source tree raises PermissionError) must not abort collection. for _name in ("vllm", "sglang"): try: importlib.import_module(_name) except Exception as _exc: # noqa: BLE001 - best-effort seeding, see above - # An engine that is simply absent is the normal case and stays quiet. - # Anything else means the install is present but broken, and every - # `import ` downstream will fail as an ordinary-looking - # collection error; one line here names the single root cause. + # A simply-absent engine is normal and stays quiet; anything else + # means the install is broken, and one line here names the cause. if not (isinstance(_exc, ModuleNotFoundError) and _exc.name == _name): print( f"conftest: {_name} is installed but could not be imported, so " diff --git a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml index fc79d03f588c..6af4eb9767fc 100644 --- a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml +++ b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml @@ -29,25 +29,8 @@ # 4 idle warm-standby GPUs for the joining group (scale 4 -> 8). # * RDMA / InfiniBand for the mooncake weight-transfer backend. Set # --mooncake-ib-device to the node's IB device list before applying. -# * A mooncake-transfer-engine build that ships the torch ProcessGroup -# extension. --elastic-ep-backend mooncake is not just the weight transport: -# SGLang registers "mooncake" (device) and "mooncake-cpu" (CPU) as -# torch.distributed backends and builds the elastic-EP process groups from -# them, so every collective those groups run -- including the all-gather -# that --enable-dp-attention issues to sync MLP batch metadata -- goes -# through it. Two ways the image can be wrong: -# - The extension is compiled per torch release (pg___, -# e.g. pg_2_11_0). A wheel with no module matching the image's exact -# torch version has no usable backend. -# - mooncake renamed the module mooncake.ep -> mooncake.pg; SGLang -# v0.5.16 imports the old name and v0.5.18 the new one, so the engine -# and the wheel have to agree on which one exists. -# While parsing arguments the Dynamo SGLang worker tries both module names -# and refuses to start with a diagnostic if neither resolves, so the first -# case fails fast instead of after the model loads. It does NOT catch the -# second: either name resolving is accepted, so a wheel that ships only the -# name this engine does not import still passes the check and then fails -# inside the engine. Match the wheel to the engine version by hand. +# * A mooncake-transfer-engine build whose torch ProcessGroup extension matches +# the image's exact torch; match mooncake.pg/mooncake.ep to the engine by hand. # * DYN_SYSTEM_ENABLED=true so the leader exposes the :9090 system server that # serves /engine/control/*. # From 343dfaf30fd142b76721d7f8fd7c01d4ac4034a9 Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Tue, 8 Sep 2026 07:43:11 +0000 Subject: [PATCH 04/11] fix(sglang): check the mooncake backend before any model I/O Run the elastic-EP preflight straight off the parsed flags, immediately after argument parsing. Reading it off ServerArgs ran it after the model download that ServerArgs.from_cli_args performs, and the diffusion and video paths build a stub namespace carrying neither flag, so both left the check inert. Narrow the ProcessGroup probe to the extension module the installed engine actually imports. Accepting either half of the mooncake.ep -> mooncake.pg rename let an image whose wheel ships only the other name pass the preflight and still fail during engine startup. Signed-off-by: svc-glamr@nvidia.com --- components/src/dynamo/sglang/args.py | 13 ++-- .../src/dynamo/sglang/elastic_ep_preflight.py | 59 ++++++++++---- .../sglang/tests/test_elastic_ep_preflight.py | 76 +++++++++++++++---- 3 files changed, 116 insertions(+), 32 deletions(-) diff --git a/components/src/dynamo/sglang/args.py b/components/src/dynamo/sglang/args.py index 7b85b12317cb..a0e639013702 100644 --- a/components/src/dynamo/sglang/args.py +++ b/components/src/dynamo/sglang/args.py @@ -442,6 +442,14 @@ async def parse_args(args: list[str]) -> Config: args_dict["disaggregation_bootstrap_port"] = bootstrap_port parsed_args = Namespace(**args_dict) + # Read off the parsed flags rather than ServerArgs: ServerArgs.from_cli_args + # downloads the model, and the diffusion and video paths build a stub that + # carries neither flag, so both would leave this unchecked. + check_elastic_ep_backend( + getattr(parsed_args, "elastic_ep_backend", None), + getattr(parsed_args, "enable_dp_attention", False), + ) + # Dynamo argument processing # If an endpoint is provided, validate and use it # otherwise fall back to default endpoints @@ -629,11 +637,6 @@ async def parse_args(args: list[str]) -> Config: if get_sglang_model_config(server_args).is_multimodal: ensure_sglang_tensor_image_size() - check_elastic_ep_backend( - getattr(server_args, "elastic_ep_backend", None), - getattr(server_args, "enable_dp_attention", False), - ) - if getattr(server_args, "schedule_low_priority_values_first", False): raise ValueError( "--schedule-low-priority-values-first is not supported in Dynamo's " diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py index 4fd1b21a9083..1677cd28de23 100644 --- a/components/src/dynamo/sglang/elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -4,18 +4,10 @@ """Startup check for SGLang's ``--elastic-ep-backend mooncake``. SGLang builds its elastic-EP process groups from the mooncake transfer -engine's torch ``ProcessGroup`` extension: ``backend="mooncake"`` for the -device group and ``backend="mooncake-cpu"`` for the CPU group. That import -and that group construction happen deep inside engine startup, well after -the model is loaded, so a worker whose image cannot serve the requested -backend only finds out late and reports it from inside upstream engine code -that the deployer did not write. Checking the same preconditions while -Dynamo is still assembling ``ServerArgs`` turns that into an argument error -naming the flag that caused it. - -This module deliberately imports no ``sglang`` symbol, and imports ``torch`` -only inside the probe helpers. That keeps it usable -- and unit-testable -- -in an image where the engine is not importable. +engine's torch ``ProcessGroup`` extension, deep inside engine startup and +after the model is loaded. The check has to run before model load to be worth +anything, and it must not import ``sglang`` or eagerly import ``torch``, so it +still answers in an image whose engine is the broken part. """ from __future__ import annotations @@ -32,9 +24,17 @@ _MOONCAKE_DEVICE_BACKEND = "mooncake" # mooncake renamed this extension ``mooncake.ep`` -> ``mooncake.pg``; SGLang -# v0.5.16 imports the old name and v0.5.18 the new one, so try both. +# v0.5.16 imports the old name and v0.5.18 the new one. _PROCESS_GROUP_MODULES = ("mooncake.pg", "mooncake.ep") +# Where SGLang names that extension. Read as text: importing them to ask would +# need the working engine this check exists to doubt, and would pull in torch +# and CUDA before it can answer. +_SGLANG_ELASTIC_EP_SOURCES = ( + "srt/elastic_ep/elastic_ep.py", + "srt/distributed/parallel_state.py", +) + # Both distributions ship the same extension; only one is normally installed. _MOONCAKE_DISTRIBUTIONS = ( "mooncake-transfer-engine-cuda13", @@ -46,6 +46,37 @@ _EXTENSION_PREFIXES = ("pg_", "ep_") +def _required_process_group_modules() -> Tuple[str, ...]: + """The extension names the installed SGLang actually imports. + + An image can hold the wrong half of the ``mooncake.ep`` -> ``mooncake.pg`` + rename: the wheel imports cleanly and the engine still cannot start, + because it asks for the other name. Reading the engine's own import line + pins which name has to work. Falls back to accepting either when those + sources cannot be read, since refusing a worker over an upstream file move + would be worse than the late failure this check replaces. + """ + try: + spec = importlib.util.find_spec("sglang") + except Exception: # noqa: BLE001 - a broken engine must not mask the real error + return _PROCESS_GROUP_MODULES + if spec is None or not spec.submodule_search_locations: + return _PROCESS_GROUP_MODULES + + required: List[str] = [] + for location in spec.submodule_search_locations: + for relative in _SGLANG_ELASTIC_EP_SOURCES: + try: + with open(os.path.join(location, relative), encoding="utf-8") as handle: + source = handle.read() + except OSError: + continue + for module_name in _PROCESS_GROUP_MODULES: + if module_name in source and module_name not in required: + required.append(module_name) + return tuple(required) or _PROCESS_GROUP_MODULES + + def _import_process_group_extension() -> Optional[str]: """Return ``None`` when the extension imports, else the collected failures. @@ -54,7 +85,7 @@ def _import_process_group_extension() -> Optional[str]: ``pg_`` module matches) before any GPU work has started. """ failures: List[str] = [] - for module_name in _PROCESS_GROUP_MODULES: + for module_name in _required_process_group_modules(): try: importlib.import_module(module_name) return None diff --git a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py index b09b856bdb49..b29270a9e652 100644 --- a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py @@ -10,6 +10,7 @@ makes over their results is what is exercised here. """ +import importlib.util import sys import pytest @@ -119,7 +120,7 @@ def test_reports_missing_mooncake_install_explicitly(monkeypatch): assert "none installed" in str(excinfo.value) -@pytest.mark.parametrize("requested_backend", [None, "", "nccl", "nvshmem"]) +@pytest.mark.parametrize("requested_backend", [None, "nccl"]) def test_ignores_backends_other_than_mooncake(monkeypatch, requested_backend): """Negative control: the same broken image is fine for everyone else. @@ -154,16 +155,16 @@ def test_unreadable_torch_backend_registry_does_not_block_startup(monkeypatch): def test_import_probe_reports_every_module_name_it_tried(monkeypatch): - """The probe itself, not a stand-in for it. - - The other cases replace the probes to pin the decision logic, which leaves - the probes themselves untested. This one runs the real - ``_import_process_group_extension`` against an import system that has no - mooncake in it, so the import, the exception capture, and the message - assembly are all the shipped code. Blocking at ``sys.meta_path`` rather - than by uninstalling makes it behave the same in an image that does have a - working wheel. + """The real probe, run against an import system with no mooncake in it. + + Blocking at ``sys.meta_path`` rather than by uninstalling makes the case + behave the same in an image that does ship a working wheel. """ + monkeypatch.setattr( + elastic_ep_preflight, + "_required_process_group_modules", + lambda: ("mooncake.pg", "mooncake.ep"), + ) class _RefuseMooncake: def find_spec(self, name, path=None, target=None): @@ -188,6 +189,55 @@ def find_spec(self, name, path=None, target=None): assert "mooncake.ep" in failure -def test_absent_mooncake_renders_as_an_explicit_absence(): - """No mooncake installed must read as words, not an empty field.""" - assert elastic_ep_preflight._format_versions({}) == "none installed" +def _install_fake_sglang_sources(monkeypatch, tmp_path, import_line): + """Point the resolver at a source tree naming one ProcessGroup module.""" + package = tmp_path / "sglang" + elastic_ep = package / "srt" / "elastic_ep" + elastic_ep.mkdir(parents=True) + (elastic_ep / "elastic_ep.py").write_text(import_line, encoding="utf-8") + + spec = importlib.util.spec_from_file_location( + "sglang", package / "__init__.py", submodule_search_locations=[str(package)] + ) + monkeypatch.setattr( + elastic_ep_preflight.importlib.util, + "find_spec", + lambda name: spec if name == "sglang" else None, + ) + + +@pytest.mark.parametrize( + "import_line, expected", + [ + ("from mooncake.pg import MooncakeBackendOptions\n", ("mooncake.pg",)), + ("from mooncake.ep import MooncakeBackendOptions\n", ("mooncake.ep",)), + ], +) +def test_probes_only_the_module_the_installed_engine_imports( + monkeypatch, tmp_path, import_line, expected +): + """The half of the rename the engine does not use must not satisfy the check. + + Accepting either name lets an image whose wheel ships only the other one + pass here and still fail during engine startup, which is the failure this + module exists to move earlier. + """ + _install_fake_sglang_sources(monkeypatch, tmp_path, import_line) + + assert elastic_ep_preflight._required_process_group_modules() == expected + + +def test_unreadable_engine_sources_probe_both_module_names(monkeypatch): + """An engine that cannot be located is not evidence for either name. + + Narrowing on a guess would refuse workers over an upstream file move, so + the resolver widens back to both names instead. + """ + monkeypatch.setattr( + elastic_ep_preflight.importlib.util, "find_spec", lambda name: None + ) + + assert elastic_ep_preflight._required_process_group_modules() == ( + "mooncake.pg", + "mooncake.ep", + ) From 0594bfaa0e7a79aa3ea1a037fd22999c341d63d0 Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Tue, 8 Sep 2026 07:59:44 +0000 Subject: [PATCH 05/11] fix(sglang): require both mooncake process-group backends at preflight SGLang builds an elastic-EP device group and a CPU group for the metadata collectives. Accepting on the device backend alone let a wheel that registered only that one pass the check and still fail during engine startup. Signed-off-by: glamr-agent Signed-off-by: svc-glamr@nvidia.com --- .../src/dynamo/sglang/elastic_ep_preflight.py | 31 ++++++++++++++----- .../sglang/tests/test_elastic_ep_preflight.py | 27 ++++++++++++++++ .../templates/sglang/sglang_elastic_ep.yaml | 3 +- 3 files changed, 52 insertions(+), 9 deletions(-) diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py index 1677cd28de23..19c1a8db0e6b 100644 --- a/components/src/dynamo/sglang/elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -19,9 +19,12 @@ MOONCAKE_BACKEND = "mooncake" -# The torch backend name SGLang registers the elastic-EP *device* group under -# (sglang/srt/distributed/parallel_state.py). "mooncake-cpu" is not required. -_MOONCAKE_DEVICE_BACKEND = "mooncake" +# The torch backend names SGLang registers its elastic-EP process groups under +# (sglang/srt/distributed/parallel_state.py): the device group and the CPU group +# it uses for the metadata collectives. The extension registers both together, so +# one present without the other is a partial registration the engine falls over +# on later. +_REQUIRED_TORCH_BACKENDS = ("mooncake", "mooncake-cpu") # mooncake renamed this extension ``mooncake.ep`` -> ``mooncake.pg``; SGLang # v0.5.16 imports the old name and v0.5.18 the new one. @@ -169,6 +172,7 @@ def _format_versions(versions: Dict[str, str]) -> str: def _build_diagnostic( import_failure: Optional[str], registered_backends: Dict[str, Tuple[str, ...]], + missing_backends: Tuple[str, ...], enable_dp_attention: bool, ) -> str: mooncake_backends = { @@ -188,6 +192,10 @@ def _build_diagnostic( f" ProcessGroup extensions this mooncake ships: {', '.join(_available_extension_modules()) or 'none found'}", f" mooncake backends registered with torch.distributed: {mooncake_backends or 'none'}", ] + if missing_backends: + lines.append( + " required by SGLang but not registered: " f"{', '.join(missing_backends)}" + ) if enable_dp_attention: lines.append( " --enable-dp-attention is also set: DP attention synchronizes its " @@ -215,7 +223,7 @@ def check_elastic_ep_backend( Raises: ValueError: The mooncake backend was requested but its torch ProcessGroup extension does not import, or imports without - registering the device backend SGLang asks torch for. + registering both of the backends SGLang asks torch for. """ if not elastic_ep_backend: return @@ -224,16 +232,23 @@ def check_elastic_ep_backend( import_failure = _import_process_group_extension() registered_backends: Dict[str, Tuple[str, ...]] = {} + missing_backends: Tuple[str, ...] = () if import_failure is None: readable_backends = _registered_torch_backends() - # An unreadable registry fails open: the import above already catches - # the reported failure, and torch could rename a private attribute. if readable_backends is None: return - if _MOONCAKE_DEVICE_BACKEND in readable_backends: + missing_backends = tuple( + name for name in _REQUIRED_TORCH_BACKENDS if name not in readable_backends + ) + if not missing_backends: return registered_backends = readable_backends raise ValueError( - _build_diagnostic(import_failure, registered_backends, enable_dp_attention) + _build_diagnostic( + import_failure, + registered_backends, + missing_backends, + enable_dp_attention, + ) ) diff --git a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py index b29270a9e652..db7965a44daa 100644 --- a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py @@ -139,6 +139,33 @@ def test_accepts_mooncake_when_the_image_can_serve_it(monkeypatch): check_elastic_ep_backend("mooncake", True) +def test_rejects_a_wheel_that_registers_only_half_the_process_groups(monkeypatch): + """A device backend without its CPU peer is not a usable image. + + SGLang builds both an elastic-EP device group and a CPU group for the + metadata collectives, so accepting on ``mooncake`` alone lets a wheel that + registered only that one pass here and still fail during engine startup. + """ + _simulate_environment( + monkeypatch, + import_failure=None, + registered_backends={ + "gloo": ("cpu",), + "nccl": ("cuda",), + "mooncake": ("cuda",), + }, + mooncake_versions={"mooncake-transfer-engine-cuda13": "0.3.11.post1"}, + extension_modules=["pg_2_11_0"], + ) + + with pytest.raises(ValueError) as excinfo: + check_elastic_ep_backend("mooncake") + + # The operator has to be told which half is absent; the registry line alone + # shows a mooncake backend present and reads as a working image. + assert "mooncake-cpu" in str(excinfo.value) + + def test_unreadable_torch_backend_registry_does_not_block_startup(monkeypatch): """A registry that cannot be read is not evidence that mooncake is absent. diff --git a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml index 6af4eb9767fc..5974a005e7d0 100644 --- a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml +++ b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml @@ -30,7 +30,8 @@ # * RDMA / InfiniBand for the mooncake weight-transfer backend. Set # --mooncake-ib-device to the node's IB device list before applying. # * A mooncake-transfer-engine build whose torch ProcessGroup extension matches -# the image's exact torch; match mooncake.pg/mooncake.ep to the engine by hand. +# the image's exact torch. The worker checks this at startup and names the +# mismatch, rather than failing from inside engine code after model load. # * DYN_SYSTEM_ENABLED=true so the leader exposes the :9090 system server that # serves /engine/control/*. # From 4a32af2aab057341673a7539e1805a991e442015 Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Tue, 8 Sep 2026 08:18:53 +0000 Subject: [PATCH 06/11] refactor(sglang): tighten engine-seeding and preflight imports Narrow conftest engine seeding to ImportError and OSError so a fault inside an engine that did start importing propagates rather than being printed. Lift the importlib.metadata imports in the mooncake preflight to module scope, leaving only the mooncake and torch probes lazy. Signed-off-by: glamr-agent Signed-off-by: svc-glamr@nvidia.com --- components/src/dynamo/sglang/args.py | 5 ++++- components/src/dynamo/sglang/elastic_ep_preflight.py | 5 ++--- conftest.py | 6 +++++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/components/src/dynamo/sglang/args.py b/components/src/dynamo/sglang/args.py index a0e639013702..d360098b1b11 100644 --- a/components/src/dynamo/sglang/args.py +++ b/components/src/dynamo/sglang/args.py @@ -444,7 +444,10 @@ async def parse_args(args: list[str]) -> Config: # Read off the parsed flags rather than ServerArgs: ServerArgs.from_cli_args # downloads the model, and the diffusion and video paths build a stub that - # carries neither flag, so both would leave this unchecked. + # carries neither flag, so both would leave this unchecked. getattr rather + # than direct access because this file supports SGLang N and N-1, and an + # older release that has not declared --elastic-ep-backend would otherwise + # raise AttributeError for every worker, not just the ones that asked for it. check_elastic_ep_backend( getattr(parsed_args, "elastic_ep_backend", None), getattr(parsed_args, "enable_dp_attention", False), diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py index 19c1a8db0e6b..84e2d6c78721 100644 --- a/components/src/dynamo/sglang/elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -15,6 +15,8 @@ import importlib import importlib.util import os +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _distribution_version from typing import Dict, List, Optional, Tuple MOONCAKE_BACKEND = "mooncake" @@ -99,9 +101,6 @@ def _import_process_group_extension() -> Optional[str]: def _installed_mooncake_versions() -> Dict[str, str]: """Map each installed mooncake distribution to its version.""" - from importlib.metadata import PackageNotFoundError - from importlib.metadata import version as _distribution_version - found: Dict[str, str] = {} for distribution in _MOONCAKE_DISTRIBUTIONS: try: diff --git a/conftest.py b/conftest.py index 747089941f8a..60a8801e35f7 100644 --- a/conftest.py +++ b/conftest.py @@ -40,7 +40,11 @@ for _name in ("vllm", "sglang"): try: importlib.import_module(_name) - except Exception as _exc: # noqa: BLE001 - best-effort seeding, see above + # ImportError is a missing engine; OSError is one whose sources are present + # but unreadable. A failure of any other kind comes from inside an engine + # that did start importing, and swallowing it here would run the suite + # against a half-initialized engine, so it propagates. + except (ImportError, OSError) as _exc: # A simply-absent engine is normal and stays quiet; anything else # means the install is broken, and one line here names the cause. if not (isinstance(_exc, ModuleNotFoundError) and _exc.name == _name): From b93167b6ed0d28f7a85048c6ec715fc30e1524b8 Mon Sep 17 00:00:00 2001 From: "svc-glamr@nvidia.com" Date: Fri, 11 Sep 2026 00:58:14 +0000 Subject: [PATCH 07/11] fix(sglang): narrow the elastic-EP preflight exception handling Catch the errors each probe can actually produce instead of bare Exception, and keep a torch that is installed but unusable out of the "not installed" bucket in the diagnostic. The one remaining broad catch is the extension import itself, where any failure is the answer, and it now carries the reason in the code rather than a terse noqa. Copying torch's backend-capability mapping into tuples also fixes the mypy arg-type error the annotation was hiding. Signed-off-by: svc-glamr@nvidia.com --- components/src/dynamo/sglang/args.py | 11 ++--- .../src/dynamo/sglang/elastic_ep_preflight.py | 47 ++++++++++++++++--- .../sglang/tests/test_elastic_ep_preflight.py | 30 +++++++++--- conftest.py | 17 +++---- 4 files changed, 79 insertions(+), 26 deletions(-) diff --git a/components/src/dynamo/sglang/args.py b/components/src/dynamo/sglang/args.py index d360098b1b11..e4e5bcbcf395 100644 --- a/components/src/dynamo/sglang/args.py +++ b/components/src/dynamo/sglang/args.py @@ -444,13 +444,12 @@ async def parse_args(args: list[str]) -> Config: # Read off the parsed flags rather than ServerArgs: ServerArgs.from_cli_args # downloads the model, and the diffusion and video paths build a stub that - # carries neither flag, so both would leave this unchecked. getattr rather - # than direct access because this file supports SGLang N and N-1, and an - # older release that has not declared --elastic-ep-backend would otherwise - # raise AttributeError for every worker, not just the ones that asked for it. + # carries neither flag, so both would leave this unchecked. parsed_args + # comes from ServerArgs.add_cli_args, which declares both options across + # the supported SGLang releases. check_elastic_ep_backend( - getattr(parsed_args, "elastic_ep_backend", None), - getattr(parsed_args, "enable_dp_attention", False), + parsed_args.elastic_ep_backend, + parsed_args.enable_dp_attention, ) # Dynamo argument processing diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py index 84e2d6c78721..0122b634016d 100644 --- a/components/src/dynamo/sglang/elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -61,9 +61,14 @@ def _required_process_group_modules() -> Tuple[str, ...]: sources cannot be read, since refusing a worker over an upstream file move would be worse than the late failure this check replaces. """ + # find_spec raises ImportError when a parent package fails to import and + # ValueError when the module is already in sys.modules without a spec. + # Both mean "cannot tell", which is what the widened fallback answers. Any + # other error is not something this fallback was reasoned about, so it + # propagates rather than silently widening the check. try: spec = importlib.util.find_spec("sglang") - except Exception: # noqa: BLE001 - a broken engine must not mask the real error + except (ImportError, ValueError): return _PROCESS_GROUP_MODULES if spec is None or not spec.submodule_search_locations: return _PROCESS_GROUP_MODULES @@ -94,7 +99,18 @@ def _import_process_group_extension() -> Optional[str]: try: importlib.import_module(module_name) return None - except Exception as exc: # noqa: BLE001 - any failure means "unusable" + # Deliberately broad. This is the one place that runs the compiled + # ProcessGroup extension's own module initialization, and the question + # being asked is "does SGLang's import of this succeed", not "does it + # fail in a way we predicted". A mismatched build surfaces as + # ImportError, but the same import also runs C++ static initializers + # and dlopens CUDA, which report as OSError, RuntimeError, or a + # torch-defined exception depending on the wheel. Narrowing here would + # let the unlisted ones escape as a raw traceback from argument parsing + # instead of the diagnostic below, which is the failure mode this whole + # module exists to remove. The exception type is kept verbatim in the + # message so nothing about the real cause is lost. + except Exception as exc: # noqa: BLE001 failures.append(f"{module_name}: {type(exc).__name__}: {exc}") return "; ".join(failures) if failures else "no attempt was made" @@ -118,7 +134,9 @@ def _available_extension_modules() -> List[str]: """ try: spec = importlib.util.find_spec("mooncake") - except Exception: # noqa: BLE001 - a broken package must not mask the real error + except (ImportError, ValueError): + # Same two failure modes as the SGLang resolver above. Either way the + # wheel's layout cannot be listed, which this reports as "none found". return [] if spec is None or not spec.submodule_search_locations: return [] @@ -145,21 +163,38 @@ def _registered_torch_backends() -> Optional[Dict[str, Tuple[str, ...]]]: transport, which is more fatal than the failure being guarded against. ``None`` keeps the two apart so the caller can skip only this assertion. """ + # ImportError covers a torch that is absent or does not import; + # AttributeError covers the rename this docstring is about. Both are + # "unreadable". torch types the mapping's values as lists, so copy them + # into tuples rather than asserting the annotation over torch's. try: import torch.distributed as torch_distributed - return dict(torch_distributed.Backend.backend_capability) - except Exception: # noqa: BLE001 - unreadable, not "nothing registered" + return { + name: tuple(devices) + for name, devices in torch_distributed.Backend.backend_capability.items() + } + except (ImportError, AttributeError): return None def _torch_version() -> str: + """The running torch version, or why it could not be determined. + + Only a genuinely absent distribution may be reported as "not installed": a + torch that is present but fails to load is a different problem, and an + operator reading this diagnostic needs to see which one they have. + """ try: import torch return str(torch.__version__) - except Exception: # noqa: BLE001 + except ModuleNotFoundError: return "not installed" + # A torch that is installed but unusable raises from its own loader, most + # often ImportError or OSError for a missing CUDA runtime library. + except (ImportError, OSError) as exc: + return f"installed but not importable: {type(exc).__name__}: {exc}" def _format_versions(versions: Dict[str, str]) -> str: diff --git a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py index db7965a44daa..328b62cb4ab1 100644 --- a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py @@ -10,6 +10,7 @@ makes over their results is what is exercised here. """ +import builtins import importlib.util import sys @@ -254,17 +255,34 @@ def test_probes_only_the_module_the_installed_engine_imports( assert elastic_ep_preflight._required_process_group_modules() == expected -def test_unreadable_engine_sources_probe_both_module_names(monkeypatch): - """An engine that cannot be located is not evidence for either name. +def test_unreadable_engine_sources_probe_both_module_names(monkeypatch, tmp_path): + """Sources that are present but unreadable are not evidence for either name. - Narrowing on a guess would refuse workers over an upstream file move, so - the resolver widens back to both names instead. + This is the case the resolver is actually written for: an editable install + whose engine tree the worker's uid cannot read, which is the same condition + the repository ``conftest.py`` reports at collection time. Narrowing on a + guess would refuse workers over a permissions problem, so the resolver + widens back to both names instead. """ - monkeypatch.setattr( - elastic_ep_preflight.importlib.util, "find_spec", lambda name: None + _install_fake_sglang_sources( + monkeypatch, tmp_path, "from mooncake.pg import MooncakeBackendOptions\n" ) + probed = [] + real_open = builtins.open + + def _refuse_engine_sources(file, *args, **kwargs): + if str(file).endswith(".py") and "sglang" in str(file): + probed.append(str(file)) + raise PermissionError(13, "Permission denied", str(file)) + return real_open(file, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", _refuse_engine_sources) + assert elastic_ep_preflight._required_process_group_modules() == ( "mooncake.pg", "mooncake.ep", ) + # The fallback must come from the read failing, not from the resolver + # skipping the sources it was pointed at. + assert probed diff --git a/conftest.py b/conftest.py index 60a8801e35f7..2ce02d84c9c8 100644 --- a/conftest.py +++ b/conftest.py @@ -45,14 +45,15 @@ # that did start importing, and swallowing it here would run the suite # against a half-initialized engine, so it propagates. except (ImportError, OSError) as _exc: - # A simply-absent engine is normal and stays quiet; anything else - # means the install is broken, and one line here names the cause. - if not (isinstance(_exc, ModuleNotFoundError) and _exc.name == _name): - print( - f"conftest: {_name} is installed but could not be imported, so " - f"tests that import it will fail: {type(_exc).__name__}: {_exc}", - file=sys.stderr, - ) + if isinstance(_exc, ModuleNotFoundError) and _exc.name == _name: + continue # the engine is simply not installed; nothing to report + # Anything else means the install is broken, and one line here names + # the cause rather than leaving a bare collection error downstream. + print( + f"conftest: {_name} is installed but could not be imported, so " + f"tests that import it will fail: {type(_exc).__name__}: {_exc}", + file=sys.stderr, + ) # Suppress ImportPathMismatchError when pytest later loads dynamo.vllm # under the bare name "vllm". From 17da01d0bbad2fe08a9e9818276e129b59f2b9b6 Mon Sep 17 00:00:00 2001 From: GLAMR Date: Fri, 11 Sep 2026 22:13:58 +0000 Subject: [PATCH 08/11] fix(sglang): clarify mooncake preflight diagnostics Signed-off-by: GLAMR --- .../src/dynamo/sglang/elastic_ep_preflight.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py index 0122b634016d..48dc6f834d88 100644 --- a/components/src/dynamo/sglang/elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -1,13 +1,10 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Startup check for SGLang's ``--elastic-ep-backend mooncake``. +"""Validate SGLang's mooncake elastic-EP backend before model loading. -SGLang builds its elastic-EP process groups from the mooncake transfer -engine's torch ``ProcessGroup`` extension, deep inside engine startup and -after the model is loaded. The check has to run before model load to be worth -anything, and it must not import ``sglang`` or eagerly import ``torch``, so it -still answers in an image whose engine is the broken part. +The check avoids importing ``sglang`` or eagerly importing ``torch`` so it can +diagnose a broken engine image. """ from __future__ import annotations @@ -189,8 +186,10 @@ def _torch_version() -> str: import torch return str(torch.__version__) - except ModuleNotFoundError: - return "not installed" + except ModuleNotFoundError as exc: + if exc.name == "torch": + return "not installed" + return f"installed but not importable: {type(exc).__name__}: {exc}" # A torch that is installed but unusable raises from its own loader, most # often ImportError or OSError for a missing CUDA runtime library. except (ImportError, OSError) as exc: From 0e164e529aa65464ce70de19945b5f43f2715753 Mon Sep 17 00:00:00 2001 From: GLAMR Date: Fri, 11 Sep 2026 22:22:18 +0000 Subject: [PATCH 09/11] fix(sglang): validate mooncake backend devices Signed-off-by: GLAMR --- .../src/dynamo/sglang/elastic_ep_preflight.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py index 48dc6f834d88..215d60acccd2 100644 --- a/components/src/dynamo/sglang/elastic_ep_preflight.py +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -19,11 +19,10 @@ MOONCAKE_BACKEND = "mooncake" # The torch backend names SGLang registers its elastic-EP process groups under -# (sglang/srt/distributed/parallel_state.py): the device group and the CPU group -# it uses for the metadata collectives. The extension registers both together, so -# one present without the other is a partial registration the engine falls over -# on later. -_REQUIRED_TORCH_BACKENDS = ("mooncake", "mooncake-cpu") +# (sglang/srt/distributed/parallel_state.py): the CUDA device group and the CPU +# group it uses for metadata collectives. The extension must register both with +# their expected device capabilities, or engine startup still fails later. +_REQUIRED_TORCH_BACKENDS = {"mooncake": "cuda", "mooncake-cpu": "cpu"} # mooncake renamed this extension ``mooncake.ep`` -> ``mooncake.pg``; SGLang # v0.5.16 imports the old name and v0.5.18 the new one. @@ -227,7 +226,8 @@ def _build_diagnostic( ] if missing_backends: lines.append( - " required by SGLang but not registered: " f"{', '.join(missing_backends)}" + " required by SGLang but missing or unavailable for its expected device: " + f"{', '.join(missing_backends)}" ) if enable_dp_attention: lines.append( @@ -256,7 +256,8 @@ def check_elastic_ep_backend( Raises: ValueError: The mooncake backend was requested but its torch ProcessGroup extension does not import, or imports without - registering both of the backends SGLang asks torch for. + registering both of the backends SGLang asks torch for with their + required device capabilities. """ if not elastic_ep_backend: return @@ -271,7 +272,9 @@ def check_elastic_ep_backend( if readable_backends is None: return missing_backends = tuple( - name for name in _REQUIRED_TORCH_BACKENDS if name not in readable_backends + name + for name, required_device in _REQUIRED_TORCH_BACKENDS.items() + if required_device not in readable_backends.get(name, ()) ) if not missing_backends: return From 1d980cce281a5dd92b6de74c291c3aa36d198e75 Mon Sep 17 00:00:00 2001 From: GLAMR Date: Wed, 16 Sep 2026 15:22:43 +0000 Subject: [PATCH 10/11] fix(ci): pass Triton framework to shared test workflow Signed-off-by: GLAMR --- .github/workflows/pr.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 57936e0b4ed6..09428070b404 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -940,6 +940,7 @@ jobs: uses: $/.github/workflows/shared-test.yml with: test_suite_name: triton + framework: triton test_type: CPU Test amd_runner: prod-tester-amd-v2 target_tag_plain: ${{ needs.triton-build.outputs.target_tag_plain }} From d20effcced933632b3af2181bcd7dcf67fe7c58e Mon Sep 17 00:00:00 2001 From: GLAMR Date: Wed, 16 Sep 2026 18:12:28 +0000 Subject: [PATCH 11/11] fix(ci): install networkx for router test collection Signed-off-by: GLAMR --- container/deps/requirements.test.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/container/deps/requirements.test.txt b/container/deps/requirements.test.txt index 15a6091a0809..a9153b3848da 100644 --- a/container/deps/requirements.test.txt +++ b/container/deps/requirements.test.txt @@ -45,6 +45,8 @@ mistral-common>=1.11.2 mypy==1.18.2 # For NATS object store verification in router tests nats-py==2.12.0 +# Router benchmark tests import graph synthesis during whole-tree collection. +networkx openai==2.32.0 # for Fault Tolerance migration tests # Protobuf definitions the OTLP parity receiver decodes; see grpcio above. # Version-matched to the opentelemetry-* pins in requirements.common.txt.