diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 57936e0b4ed6..09428070b404 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -940,6 +940,7 @@ jobs: uses: $/.github/workflows/shared-test.yml with: test_suite_name: triton + framework: triton test_type: CPU Test amd_runner: prod-tester-amd-v2 target_tag_plain: ${{ needs.triton-build.outputs.target_tag_plain }} diff --git a/components/src/dynamo/sglang/args.py b/components/src/dynamo/sglang/args.py index 769c3188e746..4f2594b6c644 100644 --- a/components/src/dynamo/sglang/args.py +++ b/components/src/dynamo/sglang/args.py @@ -40,6 +40,7 @@ resolved_server_args, ) from dynamo.sglang.backend_args import DynamoSGLangArgGroup, DynamoSGLangConfig +from dynamo.sglang.elastic_ep_preflight import check_elastic_ep_backend configure_dynamo_logging() PREFILL_DECODE_DISAGGREGATION_MODE = "pd" @@ -441,6 +442,16 @@ async def parse_args(args: list[str]) -> Config: args_dict["disaggregation_bootstrap_port"] = bootstrap_port parsed_args = Namespace(**args_dict) + # Read off the parsed flags rather than ServerArgs: ServerArgs.from_cli_args + # downloads the model, and the diffusion and video paths build a stub that + # carries neither flag, so both would leave this unchecked. parsed_args + # comes from ServerArgs.add_cli_args, which declares both options across + # the supported SGLang releases. + check_elastic_ep_backend( + parsed_args.elastic_ep_backend, + parsed_args.enable_dp_attention, + ) + # Dynamo argument processing # If an endpoint is provided, validate and use it # otherwise fall back to default endpoints diff --git a/components/src/dynamo/sglang/elastic_ep_preflight.py b/components/src/dynamo/sglang/elastic_ep_preflight.py new file mode 100644 index 000000000000..215d60acccd2 --- /dev/null +++ b/components/src/dynamo/sglang/elastic_ep_preflight.py @@ -0,0 +1,290 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate SGLang's mooncake elastic-EP backend before model loading. + +The check avoids importing ``sglang`` or eagerly importing ``torch`` so it can +diagnose a broken engine image. +""" + +from __future__ import annotations + +import importlib +import importlib.util +import os +from importlib.metadata import PackageNotFoundError +from importlib.metadata import version as _distribution_version +from typing import Dict, List, Optional, Tuple + +MOONCAKE_BACKEND = "mooncake" + +# The torch backend names SGLang registers its elastic-EP process groups under +# (sglang/srt/distributed/parallel_state.py): the CUDA device group and the CPU +# group it uses for metadata collectives. The extension must register both with +# their expected device capabilities, or engine startup still fails later. +_REQUIRED_TORCH_BACKENDS = {"mooncake": "cuda", "mooncake-cpu": "cpu"} + +# mooncake renamed this extension ``mooncake.ep`` -> ``mooncake.pg``; SGLang +# v0.5.16 imports the old name and v0.5.18 the new one. +_PROCESS_GROUP_MODULES = ("mooncake.pg", "mooncake.ep") + +# Where SGLang names that extension. Read as text: importing them to ask would +# need the working engine this check exists to doubt, and would pull in torch +# and CUDA before it can answer. +_SGLANG_ELASTIC_EP_SOURCES = ( + "srt/elastic_ep/elastic_ep.py", + "srt/distributed/parallel_state.py", +) + +# Both distributions ship the same extension; only one is normally installed. +_MOONCAKE_DISTRIBUTIONS = ( + "mooncake-transfer-engine-cuda13", + "mooncake-transfer-engine", +) + +# Compiled ProcessGroup extensions are named for the exact torch release they +# were built against, e.g. ``pg_2_11_0`` / ``ep_2_9_1``. +_EXTENSION_PREFIXES = ("pg_", "ep_") + + +def _required_process_group_modules() -> Tuple[str, ...]: + """The extension names the installed SGLang actually imports. + + An image can hold the wrong half of the ``mooncake.ep`` -> ``mooncake.pg`` + rename: the wheel imports cleanly and the engine still cannot start, + because it asks for the other name. Reading the engine's own import line + pins which name has to work. Falls back to accepting either when those + sources cannot be read, since refusing a worker over an upstream file move + would be worse than the late failure this check replaces. + """ + # find_spec raises ImportError when a parent package fails to import and + # ValueError when the module is already in sys.modules without a spec. + # Both mean "cannot tell", which is what the widened fallback answers. Any + # other error is not something this fallback was reasoned about, so it + # propagates rather than silently widening the check. + try: + spec = importlib.util.find_spec("sglang") + except (ImportError, ValueError): + return _PROCESS_GROUP_MODULES + if spec is None or not spec.submodule_search_locations: + return _PROCESS_GROUP_MODULES + + required: List[str] = [] + for location in spec.submodule_search_locations: + for relative in _SGLANG_ELASTIC_EP_SOURCES: + try: + with open(os.path.join(location, relative), encoding="utf-8") as handle: + source = handle.read() + except OSError: + continue + for module_name in _PROCESS_GROUP_MODULES: + if module_name in source and module_name not in required: + required.append(module_name) + return tuple(required) or _PROCESS_GROUP_MODULES + + +def _import_process_group_extension() -> Optional[str]: + """Return ``None`` when the extension imports, else the collected failures. + + Importing it is what SGLang itself does later; doing it here surfaces the + torch-version skew (``mooncake.pg`` raises ``ImportError`` when no + ``pg_`` module matches) before any GPU work has started. + """ + failures: List[str] = [] + for module_name in _required_process_group_modules(): + try: + importlib.import_module(module_name) + return None + # Deliberately broad. This is the one place that runs the compiled + # ProcessGroup extension's own module initialization, and the question + # being asked is "does SGLang's import of this succeed", not "does it + # fail in a way we predicted". A mismatched build surfaces as + # ImportError, but the same import also runs C++ static initializers + # and dlopens CUDA, which report as OSError, RuntimeError, or a + # torch-defined exception depending on the wheel. Narrowing here would + # let the unlisted ones escape as a raw traceback from argument parsing + # instead of the diagnostic below, which is the failure mode this whole + # module exists to remove. The exception type is kept verbatim in the + # message so nothing about the real cause is lost. + except Exception as exc: # noqa: BLE001 + failures.append(f"{module_name}: {type(exc).__name__}: {exc}") + return "; ".join(failures) if failures else "no attempt was made" + + +def _installed_mooncake_versions() -> Dict[str, str]: + """Map each installed mooncake distribution to its version.""" + found: Dict[str, str] = {} + for distribution in _MOONCAKE_DISTRIBUTIONS: + try: + found[distribution] = _distribution_version(distribution) + except PackageNotFoundError: + continue + return found + + +def _available_extension_modules() -> List[str]: + """Names of the compiled ProcessGroup extensions the wheel actually ships. + + Resolved from the package directory rather than by import, so this stays + cheap and side-effect free when the extension is the thing that is broken. + """ + try: + spec = importlib.util.find_spec("mooncake") + except (ImportError, ValueError): + # Same two failure modes as the SGLang resolver above. Either way the + # wheel's layout cannot be listed, which this reports as "none found". + return [] + if spec is None or not spec.submodule_search_locations: + return [] + + modules = set() + for location in spec.submodule_search_locations: + try: + entries = os.listdir(location) + except OSError: + continue + for entry in entries: + if not entry.startswith(_EXTENSION_PREFIXES): + continue + modules.add(entry.split(".", 1)[0]) + return sorted(modules) + + +def _registered_torch_backends() -> Optional[Dict[str, Tuple[str, ...]]]: + """torch's backend -> supported-device map, or ``None`` when unreadable. + + ``Backend.backend_capability`` is a torch internal. Reporting "could not + read the registry" as "mooncake is not registered" would let a rename in + somebody else's private attribute refuse every worker that asked for this + transport, which is more fatal than the failure being guarded against. + ``None`` keeps the two apart so the caller can skip only this assertion. + """ + # ImportError covers a torch that is absent or does not import; + # AttributeError covers the rename this docstring is about. Both are + # "unreadable". torch types the mapping's values as lists, so copy them + # into tuples rather than asserting the annotation over torch's. + try: + import torch.distributed as torch_distributed + + return { + name: tuple(devices) + for name, devices in torch_distributed.Backend.backend_capability.items() + } + except (ImportError, AttributeError): + return None + + +def _torch_version() -> str: + """The running torch version, or why it could not be determined. + + Only a genuinely absent distribution may be reported as "not installed": a + torch that is present but fails to load is a different problem, and an + operator reading this diagnostic needs to see which one they have. + """ + try: + import torch + + return str(torch.__version__) + except ModuleNotFoundError as exc: + if exc.name == "torch": + return "not installed" + return f"installed but not importable: {type(exc).__name__}: {exc}" + # A torch that is installed but unusable raises from its own loader, most + # often ImportError or OSError for a missing CUDA runtime library. + except (ImportError, OSError) as exc: + return f"installed but not importable: {type(exc).__name__}: {exc}" + + +def _format_versions(versions: Dict[str, str]) -> str: + if not versions: + return "none installed" + return ", ".join(f"{name} {value}" for name, value in sorted(versions.items())) + + +def _build_diagnostic( + import_failure: Optional[str], + registered_backends: Dict[str, Tuple[str, ...]], + missing_backends: Tuple[str, ...], + enable_dp_attention: bool, +) -> str: + mooncake_backends = { + name: devices + for name, devices in registered_backends.items() + if MOONCAKE_BACKEND in name + } + lines = [ + "--elastic-ep-backend mooncake was requested, but the mooncake torch " + "ProcessGroup backend is not usable in this image. SGLang builds its " + "elastic-EP process groups from that backend after the model is " + "loaded, so leaving this unchecked fails the worker late and from " + "inside engine code.", + f" mooncake ProcessGroup extension: {import_failure or 'imported, but registered no usable backend'}", + f" installed mooncake distributions: {_format_versions(_installed_mooncake_versions())}", + f" torch: {_torch_version()}", + f" ProcessGroup extensions this mooncake ships: {', '.join(_available_extension_modules()) or 'none found'}", + f" mooncake backends registered with torch.distributed: {mooncake_backends or 'none'}", + ] + if missing_backends: + lines.append( + " required by SGLang but missing or unavailable for its expected device: " + f"{', '.join(missing_backends)}" + ) + if enable_dp_attention: + lines.append( + " --enable-dp-attention is also set: DP attention synchronizes its " + "MLP batch metadata with an all-gather over this group on the very " + "first forward pass, so this combination reaches the backend " + "immediately at startup." + ) + lines.append( + "Install a mooncake-transfer-engine build whose ProcessGroup extension " + "matches the running torch, or drop --elastic-ep-backend." + ) + return "\n".join(lines) + + +def check_elastic_ep_backend( + elastic_ep_backend: Optional[str], + enable_dp_attention: bool = False, +) -> None: + """Reject ``--elastic-ep-backend mooncake`` when the image cannot serve it. + + Silent for every other configuration, including an unset backend: this must + never turn into an unconditional startup failure for workers that asked for + nothing of the sort. + + Raises: + ValueError: The mooncake backend was requested but its torch + ProcessGroup extension does not import, or imports without + registering both of the backends SGLang asks torch for with their + required device capabilities. + """ + if not elastic_ep_backend: + return + if str(elastic_ep_backend).strip().lower() != MOONCAKE_BACKEND: + return + + import_failure = _import_process_group_extension() + registered_backends: Dict[str, Tuple[str, ...]] = {} + missing_backends: Tuple[str, ...] = () + if import_failure is None: + readable_backends = _registered_torch_backends() + if readable_backends is None: + return + missing_backends = tuple( + name + for name, required_device in _REQUIRED_TORCH_BACKENDS.items() + if required_device not in readable_backends.get(name, ()) + ) + if not missing_backends: + return + registered_backends = readable_backends + + raise ValueError( + _build_diagnostic( + import_failure, + registered_backends, + missing_backends, + enable_dp_attention, + ) + ) diff --git a/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py new file mode 100644 index 000000000000..328b62cb4ab1 --- /dev/null +++ b/components/src/dynamo/sglang/tests/test_elastic_ep_preflight.py @@ -0,0 +1,288 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Behaviour of the ``--elastic-ep-backend mooncake`` startup check. + +Imports neither ``sglang`` nor any ``dynamo.sglang`` module that imports it, +so these run in an image where the engine is not installed or not importable +-- which is the only reason the check lives in its own module rather than in +``args.py``. The environment probes are the seams; the decision the check +makes over their results is what is exercised here. +""" + +import builtins +import importlib.util +import sys + +import pytest + +from dynamo.sglang import elastic_ep_preflight +from dynamo.sglang.elastic_ep_preflight import check_elastic_ep_backend + +pytestmark = [ + pytest.mark.unit, + pytest.mark.gpu_0, + pytest.mark.pre_merge, +] + +_IMPORT_FAILURE = ( + "mooncake.pg: ImportError: Mooncake PG was not built against torch==2.11.0; " + "mooncake.ep: ModuleNotFoundError: No module named 'mooncake.ep'" +) + + +def _simulate_environment( + monkeypatch, + *, + import_failure, + registered_backends, + mooncake_versions, + extension_modules, + torch_version="2.11.0+cu130", +): + """Pin every probe so the check runs against a known image shape.""" + monkeypatch.setattr( + elastic_ep_preflight, + "_import_process_group_extension", + lambda: import_failure, + ) + monkeypatch.setattr( + elastic_ep_preflight, + "_registered_torch_backends", + lambda: dict(registered_backends), + ) + monkeypatch.setattr( + elastic_ep_preflight, + "_installed_mooncake_versions", + lambda: dict(mooncake_versions), + ) + monkeypatch.setattr( + elastic_ep_preflight, + "_available_extension_modules", + lambda: list(extension_modules), + ) + monkeypatch.setattr(elastic_ep_preflight, "_torch_version", lambda: torch_version) + + +def _simulate_broken_mooncake(monkeypatch, **overrides): + """An image whose mooncake ProcessGroup extension does not load.""" + kwargs = { + "import_failure": _IMPORT_FAILURE, + "registered_backends": {"gloo": ("cpu",), "nccl": ("cuda",)}, + "mooncake_versions": {"mooncake-transfer-engine-cuda13": "0.3.11.post1"}, + "extension_modules": ["pg_2_9_1", "pg_2_10_0"], + } + kwargs.update(overrides) + _simulate_environment(monkeypatch, **kwargs) + + +def _simulate_healthy_mooncake(monkeypatch): + """An image whose mooncake ProcessGroup extension loads and registers.""" + _simulate_environment( + monkeypatch, + import_failure=None, + registered_backends={ + "gloo": ("cpu",), + "nccl": ("cuda",), + "mooncake": ("cuda",), + "mooncake-cpu": ("cpu",), + }, + mooncake_versions={"mooncake-transfer-engine-cuda13": "0.3.11.post1"}, + extension_modules=["pg_2_11_0"], + ) + + +def test_rejects_mooncake_backend_the_image_cannot_serve(monkeypatch): + """The worker fails at argument parsing, naming what the operator needs.""" + _simulate_broken_mooncake(monkeypatch) + + with pytest.raises(ValueError) as excinfo: + check_elastic_ep_backend("mooncake", True) + + message = str(excinfo.value) + assert "--elastic-ep-backend" in message + # The three facts that separate a bad wheel from a torch-version skew, so + # the failure is diagnosable without reproducing the crash. + assert "0.3.11.post1" in message + assert "2.11.0+cu130" in message + assert "pg_2_9_1" in message + # --enable-dp-attention is what drives the collective on the first forward + # pass, so it has to be called out when it is part of the configuration. + assert "--enable-dp-attention" in message + + +def test_reports_missing_mooncake_install_explicitly(monkeypatch): + """No mooncake at all reads as an absence, not a blank version field.""" + _simulate_broken_mooncake(monkeypatch, mooncake_versions={}, extension_modules=[]) + + with pytest.raises(ValueError) as excinfo: + check_elastic_ep_backend("mooncake") + + assert "none installed" in str(excinfo.value) + + +@pytest.mark.parametrize("requested_backend", [None, "nccl"]) +def test_ignores_backends_other_than_mooncake(monkeypatch, requested_backend): + """Negative control: the same broken image is fine for everyone else. + + Guards against the check becoming an unconditional startup failure for + workers that never asked for the mooncake transport. + """ + _simulate_broken_mooncake(monkeypatch) + + check_elastic_ep_backend(requested_backend, True) + + +def test_accepts_mooncake_when_the_image_can_serve_it(monkeypatch): + """Negative control: a working image is not blocked.""" + _simulate_healthy_mooncake(monkeypatch) + + check_elastic_ep_backend("mooncake", True) + + +def test_rejects_a_wheel_that_registers_only_half_the_process_groups(monkeypatch): + """A device backend without its CPU peer is not a usable image. + + SGLang builds both an elastic-EP device group and a CPU group for the + metadata collectives, so accepting on ``mooncake`` alone lets a wheel that + registered only that one pass here and still fail during engine startup. + """ + _simulate_environment( + monkeypatch, + import_failure=None, + registered_backends={ + "gloo": ("cpu",), + "nccl": ("cuda",), + "mooncake": ("cuda",), + }, + mooncake_versions={"mooncake-transfer-engine-cuda13": "0.3.11.post1"}, + extension_modules=["pg_2_11_0"], + ) + + with pytest.raises(ValueError) as excinfo: + check_elastic_ep_backend("mooncake") + + # The operator has to be told which half is absent; the registry line alone + # shows a mooncake backend present and reads as a working image. + assert "mooncake-cpu" in str(excinfo.value) + + +def test_unreadable_torch_backend_registry_does_not_block_startup(monkeypatch): + """A registry that cannot be read is not evidence that mooncake is absent. + + ``Backend.backend_capability`` is a torch internal. Reading it and failing + must not be worth more than the extension importing cleanly, or a torch-side + rename refuses every worker on this transport. + """ + _simulate_broken_mooncake(monkeypatch, import_failure=None) + monkeypatch.setattr( + elastic_ep_preflight, "_registered_torch_backends", lambda: None + ) + + check_elastic_ep_backend("mooncake", True) + + +def test_import_probe_reports_every_module_name_it_tried(monkeypatch): + """The real probe, run against an import system with no mooncake in it. + + Blocking at ``sys.meta_path`` rather than by uninstalling makes the case + behave the same in an image that does ship a working wheel. + """ + monkeypatch.setattr( + elastic_ep_preflight, + "_required_process_group_modules", + lambda: ("mooncake.pg", "mooncake.ep"), + ) + + class _RefuseMooncake: + def find_spec(self, name, path=None, target=None): + if name == "mooncake" or name.startswith("mooncake."): + raise ImportError(f"blocked for this test: {name}") + return None + + monkeypatch.setattr(sys, "meta_path", [_RefuseMooncake(), *sys.meta_path]) + for cached in [ + name + for name in sys.modules + if name == "mooncake" or name.startswith("mooncake.") + ]: + monkeypatch.delitem(sys.modules, cached) + + failure = elastic_ep_preflight._import_process_group_extension() + + assert failure is not None + # Both names have to appear: which one an image ships is the first thing + # an operator needs to compare against the engine version. + assert "mooncake.pg" in failure + assert "mooncake.ep" in failure + + +def _install_fake_sglang_sources(monkeypatch, tmp_path, import_line): + """Point the resolver at a source tree naming one ProcessGroup module.""" + package = tmp_path / "sglang" + elastic_ep = package / "srt" / "elastic_ep" + elastic_ep.mkdir(parents=True) + (elastic_ep / "elastic_ep.py").write_text(import_line, encoding="utf-8") + + spec = importlib.util.spec_from_file_location( + "sglang", package / "__init__.py", submodule_search_locations=[str(package)] + ) + monkeypatch.setattr( + elastic_ep_preflight.importlib.util, + "find_spec", + lambda name: spec if name == "sglang" else None, + ) + + +@pytest.mark.parametrize( + "import_line, expected", + [ + ("from mooncake.pg import MooncakeBackendOptions\n", ("mooncake.pg",)), + ("from mooncake.ep import MooncakeBackendOptions\n", ("mooncake.ep",)), + ], +) +def test_probes_only_the_module_the_installed_engine_imports( + monkeypatch, tmp_path, import_line, expected +): + """The half of the rename the engine does not use must not satisfy the check. + + Accepting either name lets an image whose wheel ships only the other one + pass here and still fail during engine startup, which is the failure this + module exists to move earlier. + """ + _install_fake_sglang_sources(monkeypatch, tmp_path, import_line) + + assert elastic_ep_preflight._required_process_group_modules() == expected + + +def test_unreadable_engine_sources_probe_both_module_names(monkeypatch, tmp_path): + """Sources that are present but unreadable are not evidence for either name. + + This is the case the resolver is actually written for: an editable install + whose engine tree the worker's uid cannot read, which is the same condition + the repository ``conftest.py`` reports at collection time. Narrowing on a + guess would refuse workers over a permissions problem, so the resolver + widens back to both names instead. + """ + _install_fake_sglang_sources( + monkeypatch, tmp_path, "from mooncake.pg import MooncakeBackendOptions\n" + ) + + probed = [] + real_open = builtins.open + + def _refuse_engine_sources(file, *args, **kwargs): + if str(file).endswith(".py") and "sglang" in str(file): + probed.append(str(file)) + raise PermissionError(13, "Permission denied", str(file)) + return real_open(file, *args, **kwargs) + + monkeypatch.setattr(builtins, "open", _refuse_engine_sources) + + assert elastic_ep_preflight._required_process_group_modules() == ( + "mooncake.pg", + "mooncake.ep", + ) + # The fallback must come from the read failing, not from the resolver + # skipping the sources it was pointed at. + assert probed diff --git a/conftest.py b/conftest.py index 9095f242563b..808e6da53868 100644 --- a/conftest.py +++ b/conftest.py @@ -36,11 +36,25 @@ _REPO_ROOT = Path(__file__).resolve().parent # Seed sys.modules with the venv copies before pytest collection runs. +# Best-effort: an engine that is present but not importable (an editable install +# with an unreadable source tree raises PermissionError) must not abort collection. for _name in ("vllm", "sglang", "triton"): try: importlib.import_module(_name) - except ImportError: - pass + # ImportError is a missing engine; OSError is one whose sources are present + # but unreadable. A failure of any other kind comes from inside an engine + # that did start importing, and swallowing it here would run the suite + # against a half-initialized engine, so it propagates. + except (ImportError, OSError) as _exc: + if isinstance(_exc, ModuleNotFoundError) and _exc.name == _name: + continue # the engine is simply not installed; nothing to report + # Anything else means the install is broken, and one line here names + # the cause rather than leaving a bare collection error downstream. + print( + f"conftest: {_name} is installed but could not be imported, so " + f"tests that import it will fail: {type(_exc).__name__}: {_exc}", + file=sys.stderr, + ) # Suppress ImportPathMismatchError when pytest later loads dynamo.vllm # under the bare name "vllm". diff --git a/container/deps/requirements.test.txt b/container/deps/requirements.test.txt index 15a6091a0809..a9153b3848da 100644 --- a/container/deps/requirements.test.txt +++ b/container/deps/requirements.test.txt @@ -45,6 +45,8 @@ mistral-common>=1.11.2 mypy==1.18.2 # For NATS object store verification in router tests nats-py==2.12.0 +# Router benchmark tests import graph synthesis during whole-tree collection. +networkx openai==2.32.0 # for Fault Tolerance migration tests # Protobuf definitions the OTLP parity receiver decodes; see grpcio above. # Version-matched to the opentelemetry-* pins in requirements.common.txt. diff --git a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml index 4f7a5c281be4..5974a005e7d0 100644 --- a/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml +++ b/tests/fault_tolerance/deploy/templates/sglang/sglang_elastic_ep.yaml @@ -29,6 +29,9 @@ # 4 idle warm-standby GPUs for the joining group (scale 4 -> 8). # * RDMA / InfiniBand for the mooncake weight-transfer backend. Set # --mooncake-ib-device to the node's IB device list before applying. +# * A mooncake-transfer-engine build whose torch ProcessGroup extension matches +# the image's exact torch. The worker checks this at startup and names the +# mismatch, rather than failing from inside engine code after model load. # * DYN_SYSTEM_ENABLED=true so the leader exposes the :9090 system server that # serves /engine/control/*. #