feat(customizer): add RL backend in customizer plugin

docker-bake.hcl

@@ -327,6 +327,67 @@ group "nmp-unsloth" {
   ]
 }
 
+group "nmp-rl" {
+  targets = [
+    "nmp-rl-base-builder",
+    "nmp-rl-tasks",
+    "nmp-rl-training",
+  ]
+}
+
+# Pruned workspace slice for nmp-rl images (keep in sync with
+# docker/rl/pyproject.workspace.toml + Dockerfile.platform-workspace members).
+target "rl-platform-workspace" {
+  target     = "platform-workspace"
+  context    = "."
+  dockerfile = "docker/rl/Dockerfile.platform-workspace"
+  output     = ["type=cacheonly"]
+  platforms  = get_platforms()
+}
+
+# Heavy base: NGC torch/CUDA + NeMo-RL v0.4.0 + Ray.
+target "nmp-rl-base-builder" {
+  target     = "nmp-rl-base"
+  context    = "."
+  dockerfile = "docker/Dockerfile.nmp-rl-base"
+  cache-to   = maybe_registry_cache_to("nmp-rl-base")
+  cache-from = maybe_registry_cache_from("nmp-rl-base")
+  tags       = base_tags("nmp-rl-base")
+  output     = image_output()
+  platforms  = get_platforms()
+}
+
+# GPU DPO training image: base + platform glue. Bootstraps Ray at runtime.
+target "nmp-rl-training" {
+  target     = "runtime"
+  context    = "."
+  dockerfile = "docker/Dockerfile.nmp-rl-training"
+  contexts = {
+    platform-workspace = "target:rl-platform-workspace"
+    nmp-rl-base        = "target:nmp-rl-base-builder"
+  }
+  cache-to   = maybe_registry_cache_to("nmp-rl-training")
+  cache-from = maybe_registry_cache_from("nmp-rl-training")
+  tags       = sha_and_maybe_latest_tags("nmp-rl-training")
+  output     = image_output()
+  platforms  = get_platforms()
+}
+
+# Lighter CPU image for the file_io / model_entity steps (no NeMo-RL/Ray).
+target "nmp-rl-tasks" {
+  target     = "runtime"
+  context    = "."
+  dockerfile = "docker/Dockerfile.nmp-rl-tasks"
+  contexts = {
+    platform-workspace = "target:rl-platform-workspace"
+  }
+  cache-to   = maybe_registry_cache_to("nmp-rl-tasks")
+  cache-from = maybe_registry_cache_from("nmp-rl-tasks")
+  tags       = sha_and_maybe_latest_tags("nmp-rl-tasks")
+  output     = image_output()
+  platforms  = get_platforms()
+}
+
 # Base images for consolidated containers
 target "nmp-python-base" {
   target     = python_base_target()

docker/Dockerfile.nmp-rl-base

@@ -0,0 +1,56 @@
+# syntax=docker/dockerfile:1
+# nmp-rl base - NVIDIA NeMo-RL v0.6.0 NGC container + CVE hardening.
+#
+# Bases on the PUBLISHED NGC NeMo-RL container (amd64 + arm64 at v0.6.0), which
+# bundles the full RL stack on Python 3.13: PyTorch 2.10, Ray 2.54.0, vLLM
+# 0.17.1, Megatron-Core 0.18, Transformers 5.3.
+#
+# Publish target: nmp-rl-base.
+
+# Referenced by tag rather than a @sha256 digest pin: NGC publishes separate
+# per-architecture tags/digests for nemo-rl, so a single digest doesn't cleanly
+# cover the multi-arch (amd64 + arm64) base we build on. The :v0.6.0 tag is the
+# stable, arch-agnostic reference.
+ARG NEMO_RL_IMAGE=nvcr.io/nvidia/nemo-rl:v0.6.0
+
+FROM ${NEMO_RL_IMAGE} AS nmp-rl-base
+
+# NeMo-RL v0.6.0 ships its venv at /opt/nemo_rl_venv (Python 3.13).
+# VERIFY this path against the actual image if NeMo-RL relocates it in a future
+# tag — the training image's /opt/venv symlink and entrypoint depend on it.
+ENV VIRTUAL_ENV=/opt/nemo_rl_venv \
+    HF_HUB_ENABLE_HF_TRANSFER=1 \
+    OTEL_PYTHON_EXCLUDED_URLS="health"
+
+# The NGC base venv is built as root with locked-down cache dirs; relax them so
+# editable installs in the training image can write.
+RUN chmod 755 /root /root/.cache /root/.cache/uv /root/.local /root/.local/share /root/.local/share/uv 2>/dev/null || true
+
+# NeMo-RL is installed EDITABLE in this base: its source tree lives at
+# /opt/nemo-rl (root-owned, non-traversable) and the editable import finder
+# os.stat()s files under it at import time. The training image drops to
+# USER 1000, so `from nemo_rl... import ...` hits the finder and fails with
+# PermissionError on /opt/nemo-rl/nemo_rl/__init__.py. Make the source tree
+# world-readable + dir-traversable (a+rX) so the non-root runtime can import it.
+# Read-only is enough — nothing writes back into the editable tree at runtime.
+RUN chmod -R a+rX /opt/nemo-rl
+
+# CVE remediation: remove vLLM from the main venv and any pre-built Ray worker
+# venvs (NeMo-RL rebuilds those at runtime). vLLM is unused by DPO/SFT/LoRA.
+# Add it back when adding GRPO.
+RUN set -e; \
+    rm -rf ${VIRTUAL_ENV}/lib/python*/site-packages/vllm \
+           ${VIRTUAL_ENV}/lib/python*/site-packages/vllm-*.dist-info; \
+    for venv_dir in /opt/ray_venvs/*/; do \
+        [ -d "$venv_dir/lib" ] || continue; \
+        rm -rf "$venv_dir"/lib/python*/site-packages/vllm \
+               "$venv_dir"/lib/python*/site-packages/vllm-*.dist-info; \
+    done
+
+# Drop to a non-root user by default so DIRECT consumers of this published base do
+# not run with container root. The build steps above intentionally run as root (the
+# NGC base default — a USER directive only changes the final/default user, not the
+# RUNs preceding it). Derived images that need root for their own build steps
+# (e.g. Dockerfile.nmp-rl-training) re-assert `USER root` and then drop back to a
+# non-root UID at the end. UID 1000 is the NGC base's `ubuntu` user.
+USER 1000:1000

docker/Dockerfile.nmp-rl-tasks

@@ -0,0 +1,73 @@
+# syntax=docker/dockerfile:1
+# nmp-rl tasks - CPU file_io / model_entity steps for an nmp-rl DPO job.
+#
+# Deliberately does NOT build on nmp-rl-base: the download/upload/model-entity
+# steps only need the platform glue (SDK + customization-common), not NeMo-RL /
+# Ray / vLLM. Basing on the NGC image keeps it consistent with the platform's
+# CUDA userspace while staying far lighter than the training image.
+
+ARG SMOKE_MARKER=smoke_nmp_rl_tasks
+ARG PYTORCH_BASE=nvcr.io/nvidia/pytorch:26.02-py3
+
+FROM ${PYTORCH_BASE} AS base
+
+WORKDIR /app
+
+COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv
+ENV PATH="/bin:${PATH}"
+
+ENV VIRTUAL_ENV=/opt/venv \
+    UV_PROJECT_ENVIRONMENT=/opt/venv \
+    UV_LINK_MODE=copy \
+    UV_COMPILE_BYTECODE=1 \
+    HF_HUB_ENABLE_HF_TRANSFER=1 \
+    OTEL_PYTHON_EXCLUDED_URLS="health"
+ENV PATH="/opt/venv/bin:/root/.local/bin:${PATH}"
+
+RUN uv venv ${UV_PROJECT_ENVIRONMENT} --system-site-packages
+
+FROM base AS runtime
+
+ARG USERNAME=ubuntu
+ARG USER_UID=1000
+ARG USER_GID=1000
+
+COPY --from=platform-workspace / /app
+WORKDIR /app
+
+RUN mkdir -p /home/${USERNAME}/.cache && \
+    chown -R ${USER_UID}:${USER_GID} /home/${USERNAME} /app/services/rl
+
+# Only the glue + nmp-rl (compile/tasks side). No NeMo-RL extra → no Ray/vLLM.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache \
+    -e /app/sdk/python/nemo-platform \
+    -e /app/packages/nemo_platform_plugin \
+    -e /app/packages/nmp_common \
+    -e /app/packages/nmp_customization_common \
+    -e /app/services/rl
+
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+# Default CMD is a harmless help invocation; the platform overrides `command`
+# per step (e.g. -m nmp.rl.tasks.file_io / nmp.rl.tasks.model_entity).
+ENTRYPOINT ["/opt/venv/bin/python"]
+CMD ["-m", "nmp.rl.tasks.file_io", "--help"]
+
+USER ${USER_UID}:${USER_GID}
+
+# NOTE: this smoke-test stage is intentionally NOT wired into docker-bake.hcl
+# (unlike nmp-automodel-{tasks,training}-smoke-test). There are no RL smoke tests
+# yet — tests/smoke_gpu/ carries no tests marked `smoke_nmp_rl_tasks`, and that
+# marker isn't registered in its conftest.py. Wiring a bake target now would run
+# `pytest -m smoke_nmp_rl_tasks` against zero collected tests, which exits 5
+# ("no tests ran") and fails the bake. Add RL import smoke tests + register the
+# marker first, then add the bake target (see nmp-automodel-tasks-smoke-test for
+# the pattern). The stage is kept so it's ready to wire up once tests exist.
+FROM runtime AS smoke-test
+ARG SMOKE_MARKER
+USER 0
+COPY tests/smoke_gpu/ /smoke_test/
+RUN uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache --reinstall pytest && \
+    ${VIRTUAL_ENV}/bin/pytest /smoke_test/ -m ${SMOKE_MARKER} -v
+
+FROM runtime

docker/Dockerfile.nmp-rl-training

@@ -0,0 +1,105 @@
+# syntax=docker/dockerfile:1
+# nmp-rl training - GPU DPO step (NeMo-RL v0.6.0 + Ray + nmp-rl package).
+#
+# Built on nmp-rl-base (NGC NeMo-RL v0.6.0 venv at /opt/nemo_rl_venv, Python
+# 3.13). Adds the lightweight platform glue editably into that venv, using the
+# ported no_override_requirements.txt so the install does not clobber NeMo-RL's
+# pinned ML stack (ray / torch / mlflow / cryptography / starlette /
+# prometheus-client). The training step runs `python -m nmp.rl.tasks.training`,
+# which bootstraps a Ray cluster and runs the DPO driver against NeMo-RL.
+
+ARG SMOKE_MARKER=smoke_nmp_rl_training
+
+# Supplied by bake (target:nmp-rl-base-builder).
+FROM nmp-rl-base AS rl-base
+
+FROM rl-base AS runtime
+
+# nmp-rl-base now defaults to a non-root UID; re-assert root for the editable
+# installs, chowns and symlink below. The stage drops back to USER ${USER_UID} at
+# the end, so the published training image still runs non-root.
+USER root
+
+COPY --from=ghcr.io/astral-sh/uv:0.9.14 /uv /bin/uv
+ENV PATH="/bin:${PATH}"
+
+ARG USERNAME=ubuntu
+ARG USER_UID=1000
+ARG USER_GID=1000
+
+COPY --from=platform-workspace / /app
+WORKDIR /app
+
+RUN mkdir -p /home/${USERNAME}/.cache && \
+    chown -R ${USER_UID}:${USER_GID} /home/${USERNAME} /app/services/rl
+
+# NMP is pinned to Python 3.11, but the NeMo-RL base venv is 3.13. The glue is
+# pure-Python and installs into the 3.13 venv fine. Symlink /opt/venv ->
+# /opt/nemo_rl_venv so the single RL_PYTHON_ENTRYPOINT (/opt/venv/bin/python)
+# the compiler stamps onto every step resolves in this image too (the CPU tasks
+# image has a real /opt/venv).
+RUN ln -sfn /opt/nemo_rl_venv /opt/venv
+
+# The NeMo-RL base venv ships some packages with missing dist-info METADATA (a
+# known property of the image — the package code is present, only uv's metadata
+# file is gone). Normal `uv pip install` reads ALL installed metadata during
+# dependency resolution and dies ("Failed to read anyio==... METADATA"). So we
+# install with --no-deps, which skips resolution entirely and never touches the
+# corrupted metadata. The heavy ML stack already lives in the base venv,
+#so --no-deps leaves it intact.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache --no-deps \
+    -e /app/sdk/python/nemo-platform \
+    -e /app/packages/nemo_platform_plugin \
+    -e /app/packages/nmp_common \
+    -e /app/packages/nmp_customization_common \
+    -e /app/services/rl
+
+# Light PURE-PYTHON glue deps the NeMo-RL base lacks (or ships with corrupted
+# dist-info). Three rules:
+#   1. --reinstall: force a clean (re)install of each target. uv otherwise reads
+#      the installed copy's METADATA for its "already satisfied?" check, which
+#      fails on the base's missing-METADATA dist-info (e.g. idna, anyio). A
+#      version pin alone is NOT enough — when the pin matches the installed
+#      (corrupted) version, uv still tries to read it. --reinstall skips that
+#      check entirely and installs fresh from the index (also repairing metadata).
+#   2. --no-deps: never resolve/read OTHER packages' metadata.
+#   3. ONLY non-ML, version-stable packages. Base ML deps (pydantic, anyio,
+#      httpx, typing-extensions, …) are present + version-sensitive to the
+#      torch/transformers stack — we must NOT reinstall them, so they are omitted.
+# If the training step hits a runtime ImportError for another pure-python
+# package, add it here (pinned), NOT one of the ML deps above.
+# Discovered empirically via scripts/gpu-dpo-smoke/discover_deps.py against the
+# v0.6.0 base — the ONLY pure-python deps the training entrypoint imports that are
+# missing from /opt/nemo_rl_venv. base58 + lark are NeMo-RL's OWN deps (absent
+# from the main venv); distro is the SDK's; pydantic-settings is our glue's.
+# Re-run discover_deps.py after a base-image bump to refresh this list.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache --no-deps --reinstall \
+    "base58==2.1.1" \
+    "lark==1.3.1" \
+    "distro==1.9.0" \
+    "pydantic-settings==2.14.2"
+
+ENV PATH="${VIRTUAL_ENV}/bin:${PATH}"
+ENTRYPOINT ["/opt/venv/bin/python"]
+CMD ["-m", "nmp.rl.tasks.training", "--help"]
+
+USER ${USER_UID}:${USER_GID}
+
+# NOTE: this smoke-test stage is intentionally NOT wired into docker-bake.hcl
+# (unlike nmp-automodel-{tasks,training}-smoke-test). There are no RL smoke tests
+# yet — tests/smoke_gpu/ carries no tests marked `smoke_nmp_rl_training`, and that
+# marker isn't registered in its conftest.py. Wiring a bake target now would run
+# `pytest -m smoke_nmp_rl_training` against zero collected tests, which exits 5
+# ("no tests ran") and fails the bake. Add RL import smoke tests + register the
+# marker first, then add the bake target (see nmp-automodel-training-smoke-test
+# for the pattern). The stage is kept so it's ready to wire up once tests exist.
+FROM runtime AS smoke-test
+ARG SMOKE_MARKER
+USER 0
+COPY tests/smoke_gpu/ /smoke_test/
+RUN uv pip install --python ${VIRTUAL_ENV}/bin/python --no-cache --reinstall pytest && \
+    ${VIRTUAL_ENV}/bin/pytest /smoke_test/ -m ${SMOKE_MARKER} -v
+
+FROM runtime

docker/rl/Dockerfile.platform-workspace

@@ -0,0 +1,18 @@
+# syntax=docker/dockerfile:1
+# Minimal Platform workspace slice for nmp-rl container installs.
+# Used as a named build context (platform-workspace).
+# Keep in sync with docker/rl/pyproject.workspace.toml members.
+
+FROM scratch AS platform-workspace
+# Reduced workspace file for this partial source tree.
+COPY docker/rl/pyproject.workspace.toml pyproject.toml
+# nemo-platform-sdk's hatch build force-includes docs/ from the repo root; the
+# openapi symlink must resolve at build time, so copy both trees.
+COPY docs docs
+COPY openapi openapi
+COPY packages/nmp_build_tools packages/nmp_build_tools
+COPY packages/nmp_common packages/nmp_common
+COPY packages/nmp_customization_common packages/nmp_customization_common
+COPY packages/nemo_platform_plugin packages/nemo_platform_plugin
+COPY sdk/python/nemo-platform sdk/python/nemo-platform
+COPY services/rl services/rl

docker/rl/pyproject.workspace.toml

@@ -0,0 +1,29 @@
+# Minimal uv workspace for nmp-rl container image builds only.
+# Keeps uv validation scoped to the partial source tree copied into the image.
+# Keep in sync with docker/rl/Dockerfile.platform-workspace members.
+
+[project]
+name = "nemo-platform-rl-image"
+version = "0.0.0"
+requires-python = ">=3.11,<3.14"
+
+[tool.uv]
+required-version = ">=0.9.14,<0.10.0"
+
+[tool.uv.workspace]
+members = [
+  "packages/nmp_build_tools",
+  "sdk/python/nemo-platform",
+  "packages/nemo_platform_plugin",
+  "packages/nmp_common",
+  "packages/nmp_customization_common",
+  "services/rl",
+]
+
+[tool.uv.sources]
+nmp-build-tools = { workspace = true }
+nemo-platform-sdk = { workspace = true }
+nemo-platform-plugin = { workspace = true }
+nmp-common = { workspace = true }
+nmp-customization-common = { workspace = true }
+nmp-rl = { workspace = true }

packages/nmp_customization_common/src/nmp/customization_common/contributor/jobs.py

@@ -39,6 +39,24 @@ def require_docker_runtime(backend_label: str) -> None:
         )
 
 
+def require_distributed_runtime(backend_label: str) -> None:
+    """Refuse to compile when the platform isn't a remote Kubernetes cluster.
+
+    Sibling to :func:`require_docker_runtime` for backends that provision a Ray
+    cluster (e.g. NeMo-RL DPO). Unlike the SFT backends, these have no local
+    single-node Docker fallback: they need the platform's Kubernetes/Volcano
+    scheduler to place GPU pods and inject the distributed env
+    (``RANK``/``WORLD_SIZE``/``MASTER_ADDR``). Surface the misconfiguration before
+    the Jobs API rejects the spec.
+    """
+    platform_config = NemoPlatformConfig.get()
+    if platform_config.runtime != Runtime.KUBERNETES:
+        raise PlatformJobCompilationError(
+            f"{backend_label} training requires platform.runtime: kubernetes — it provisions a Ray "
+            "cluster on the remote GPU cluster and has no local Docker fallback.",
+        )
+
+
 class BaseSubmitJob(NemoJob):
     """Shared submit-only job scaffold.