feat(dynamo): add managed Slurm generation with W&B telemetry

.github/workflows/cicd-main.yml

@@ -224,7 +224,11 @@ jobs:
           h100_functional_test_scripts=$(jq -c \
             '[.functional[] | select((ascii_downcase | contains("gb200")) | not)]' \
             <<< "$test_plan")
-          gb200_functional_test_scripts=$(jq -c '.functional' <<< "$test_plan")
+          # The managed Dynamo runtime is currently amd64-only and is not
+          # installed in the arm64 GB200 image.
+          gb200_functional_test_scripts=$(jq -c \
+            '[.functional[] | select(. != "L1_Functional_Tests_Dynamo")]' \
+            <<< "$test_plan")
 
           unit_test_count=$(jq 'length' <<< "$unit_test_scripts")
           h100_functional_test_count=$(jq 'length' <<< "$h100_functional_test_scripts")
@@ -553,6 +557,7 @@ jobs:
       megatron-lock-artifact: ${{ needs.prepare-megatron-lock.outputs.artifact_name }}
       build-args: |
         MAX_JOBS=4
+        BUILD_DYNAMO=1
         TRTLLM_BUILD_JOBS=24
         NEMO_RL_COMMIT=${{ needs.pre-flight.outputs.test_sha }}
 

.github/workflows/lockfile-check.yml

@@ -26,6 +26,8 @@ on:
       - "pyproject.toml"
       - "uv.lock"
       - "3rdparty/**"
+      - "docker/dynamo/pyproject.toml"
+      - "docker/dynamo/uv.lock"
       - ".github/workflows/lockfile-check.yml"
 
 jobs:
@@ -48,3 +50,6 @@ jobs:
       # uv.lock is slow (~5 min); cached runs are fast.
       - name: Check lockfile is up to date
         run: uv lock --check
+
+      - name: Check Dynamo lockfile is up to date
+        run: uv lock --check --directory docker/dynamo

docker/Dockerfile

@@ -22,6 +22,8 @@
 #   --build-arg SKIP_VLLM_BUILD=1    # Skip vLLM dependencies
 #   --build-arg SKIP_SGLANG_BUILD=1  # Skip SGLang dependencies
 #   --build-arg SKIP_TRTLLM_BUILD=1  # Skip TRT-LLM dependencies
+# Optional isolated managed Dynamo runtime (Dynamo + vLLM only):
+#   --build-arg BUILD_DYNAMO=1
 
 ARG BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:26.05-cuda13.2-devel-ubuntu24.04
 FROM scratch AS nemo-rl
@@ -334,6 +336,10 @@ FROM hermetic AS release
 ARG SKIP_VLLM_BUILD
 ARG SKIP_SGLANG_BUILD
 ARG SKIP_TRTLLM_BUILD
+ARG BUILD_DYNAMO
+ARG DYNAMO_PYTHON_VERSION=3.12.11
+ARG ETCD_VERSION=v3.5.21
+ARG NATS_VERSION=v2.11.6
 # Space-separated config paths whose NeMo Gym venvs are prefetched into the image
 # (empty default = skip). Consumed by the gym prefetch RUN below.
 ARG NEMO_GYM_PREFETCH_CONFIGS=
@@ -349,6 +355,7 @@ LABEL com.nvidia.build.id="${NVIDIA_BUILD_ID}"
 LABEL com.nvidia.build.ref="${NVIDIA_BUILD_REF}"
 
 ENV NEMO_RL_VENV_DIR=/opt/ray_venvs
+ENV NEMO_RL_DYNAMO_VENV_DIR=/opt/dynamo_venv
 
 # AWS EFA OFI plugin discovery (p4d / p5 / p5en multi-node NCCL via SRD).
 # The host's aws-efa-installer places libnccl-net-ofi.so under
@@ -364,6 +371,16 @@ ENV LD_LIBRARY_PATH="/opt/amazon/ofi-nccl/lib:/opt/amazon/efa/lib:${LD_LIBRARY_P
 # Copy in source from build context (defaults to cloned repo, can be overridden)
 # Exclude pyproject.toml and uv.lock since those may be altered by build-custom-vllm.sh
 COPY --from=nemo-rl --exclude=pyproject.toml --exclude=uv.lock . /opt/nemo-rl
+
+# Keep Dynamo's Python 3.12 dependency graph isolated from NeMo-RL's normal
+# Ray/vLLM environments. Default image builds do not create this environment.
+RUN <<"EOF" bash -exu -o pipefail
+if [[ "${BUILD_DYNAMO:-0}" != "1" ]]; then
+    echo "BUILD_DYNAMO is not 1; skipping the managed Dynamo runtime"
+    exit 0
+fi
+bash docker/dynamo/install.sh
+EOF
 # Unshallow the repo to get the full history (in the case it was from the scratch layer).
 # Potentially not necessary if the repo is passed in as a complete repository (w/ full git history),
 # so do a quick check before trying to unshallow.

docker/dynamo/install.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Copyright (c) 2026, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -euo pipefail
+
+script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+project_dir=${NEMO_RL_DYNAMO_PROJECT_DIR:-${script_dir}}
+repo_root=$(realpath "${script_dir}/../..")
+dynamo_venv_dir=${NEMO_RL_DYNAMO_VENV_DIR:-${repo_root}/venvs/dynamo}
+dynamo_python_version=${DYNAMO_PYTHON_VERSION:-3.12.11}
+etcd_version=${ETCD_VERSION:-v3.5.21}
+nats_version=${NATS_VERSION:-v2.11.6}
+
+target_arch=${TARGETARCH:-}
+if [[ -z "${target_arch}" ]]; then
+  case "$(uname -m)" in
+    x86_64) target_arch=amd64 ;;
+    aarch64) target_arch=arm64 ;;
+    *)
+      echo "Unsupported host architecture: $(uname -m)" >&2
+      exit 2
+      ;;
+  esac
+fi
+case "${target_arch}" in
+  amd64|arm64) ;;
+  *)
+    echo "Unsupported TARGETARCH: ${target_arch}" >&2
+    exit 2
+    ;;
+esac
+
+uv python install "${dynamo_python_version}"
+uv venv --python "${dynamo_python_version}" "${dynamo_venv_dir}"
+UV_PROJECT_ENVIRONMENT="${dynamo_venv_dir}" uv sync \
+  --directory "${project_dir}" \
+  --locked \
+  --no-dev \
+  --no-install-project \
+  --link-mode copy
+
+dynamo_python=${dynamo_venv_dir}/bin/python
+vllm_version=$("${dynamo_python}" -c \
+  'from importlib.metadata import version; print(version("vllm"))')
+if [[ "${vllm_version}" != "0.23.0" ]]; then
+  echo "Expected vllm==0.23.0 from ai-dynamo[vllm]==1.3.0.post1; got ${vllm_version}" >&2
+  exit 1
+fi
+
+vllm_root=$("${dynamo_python}" -c \
+  'from pathlib import Path; import vllm; print(Path(vllm.__file__).resolve().parent.parent)')
+patch_file=${project_dir}/patches/vllm-0.23.0-layerwise-reload-composed-loader.patch
+
+# Dynamo 1.3.0 pins vLLM 0.23.0, which predates vLLM PR #44814.
+# Without that fix, composed weight loaders can make layerwise reload finalize
+# a layer early, leaving trailing NemotronH/Mamba2 parameters such as mixer.D
+# unloaded and corrupting logits after a weight refit.
+# Remove this backport only after Dynamo pins a vLLM release containing #44814.
+if git -C "${vllm_root}" apply --check "${patch_file}"; then
+  git -C "${vllm_root}" apply "${patch_file}"
+elif [[ -f "${dynamo_venv_dir}/VLLM_BACKPORTS" ]] \
+  && git -C "${vllm_root}" apply --reverse --check "${patch_file}"; then
+  echo "vLLM PR #44814 backport is already applied"
+else
+  echo "vLLM PR #44814 backport does not apply cleanly to vLLM ${vllm_version}" >&2
+  exit 1
+fi
+printf '%s\n' \
+  'vllm PR #44814 merge commit c9e5bf813530fb9ce06024e075da0f520b0718c8' \
+  > "${dynamo_venv_dir}/VLLM_BACKPORTS"
+
+download_dir=$(mktemp -d "${TMPDIR:-/tmp}/nemorl-dynamo-install.XXXXXX")
+trap 'rm -rf "${download_dir}"' EXIT
+
+curl --fail --location --retry 3 \
+  "https://github.com/etcd-io/etcd/releases/download/${etcd_version}/etcd-${etcd_version}-linux-${target_arch}.tar.gz" \
+  --output "${download_dir}/etcd.tgz"
+tar -xzf "${download_dir}/etcd.tgz" -C "${download_dir}"
+install -m 0755 \
+  "${download_dir}/etcd-${etcd_version}-linux-${target_arch}/etcd" \
+  "${dynamo_venv_dir}/bin/etcd"
+
+curl --fail --location --retry 3 \
+  "https://github.com/nats-io/nats-server/releases/download/${nats_version}/nats-server-${nats_version}-linux-${target_arch}.tar.gz" \
+  --output "${download_dir}/nats.tgz"
+tar -xzf "${download_dir}/nats.tgz" -C "${download_dir}"
+install -m 0755 \
+  "${download_dir}/nats-server-${nats_version}-linux-${target_arch}/nats-server" \
+  "${dynamo_venv_dir}/bin/nats-server"
+
+"${dynamo_python}" -c \
+  'import importlib.metadata as m; assert m.version("ai-dynamo") == "1.3.0.post1"; assert m.version("vllm") == "0.23.0"; assert m.version("nvidia-nccl-cu13") == "2.30.7"'
+test -s "${dynamo_venv_dir}/VLLM_BACKPORTS"
+grep -Fqx \
+  'vllm PR #44814 merge commit c9e5bf813530fb9ce06024e075da0f520b0718c8' \
+  "${dynamo_venv_dir}/VLLM_BACKPORTS"
+"${dynamo_venv_dir}/bin/etcd" --version
+"${dynamo_venv_dir}/bin/nats-server" --version

docker/dynamo/patches/vllm-0.23.0-layerwise-reload-composed-loader.patch

@@ -0,0 +1,24 @@
+diff --git a/vllm/model_executor/model_loader/reload/meta.py b/vllm/model_executor/model_loader/reload/meta.py
+index 397a458cb..ad6eceb8a 100644
+--- a/vllm/model_executor/model_loader/reload/meta.py
++++ b/vllm/model_executor/model_loader/reload/meta.py
+@@ -185,4 +185,18 @@ def get_numel_loaded(
+     """
+     with CopyCounter() as counter:
+         return_value = weight_loader(*args.args, **args.kwargs)
+-    return counter.copied_numel, return_value
++
++    # A weight loader fills a single destination parameter, so the number of
++    # loaded elements is at most that parameter's size. Some loaders copy into
++    # the parameter more than once -- e.g. ``composed_weight_loader`` runs an
++    # in-place post-load transform (``param.copy_(fn(param))``) on top of the
++    # initial copy -- which would make CopyCounter report twice the parameter
++    # size. Over-counting inflates the layer's loaded-element total and can
++    # finalize the layer before every parameter is loaded, silently dropping
++    # the trailing parameter(s) (e.g. Mamba ``mixer.D``). Cap the count at the
++    # destination size to keep the per-layer accounting correct.
++    numel = counter.copied_numel
++    param = args.arguments.get("param", None)
++    if isinstance(param, torch.Tensor):
++        numel = min(numel, param.numel())
++    return numel, return_value

docker/dynamo/pyproject.toml

@@ -0,0 +1,9 @@
+[project]
+name = "nemo-rl-dynamo-runtime"
+version = "0.0.0"
+requires-python = "==3.12.*"
+dependencies = ["ai-dynamo[vllm]==1.3.0.post1"]
+
+[tool.uv]
+package = false
+override-dependencies = ["nvidia-nccl-cu13==2.30.7; sys_platform == 'linux'"]