fix(customizer): fix 500 on validation failure for automodel, busybox default

.github/actions/setup-kind-cluster/action.yaml

@@ -163,7 +163,7 @@ runs:
         HELM_VALUES: ${{ inputs['helm-values'] }}
         REQUIRE_NMP_E2E_IMAGES: "true"
         POSTGRES_IMAGE: docker.io/library/postgres
-        BUSYBOX_IMAGE: docker.io/library/busybox
+        BUSYBOX_IMAGE: docker.io/library/busybox:stable
         GITHUB_TOKEN: ${{ inputs['kind-image-pull-token'] }}
       run: |
         if ! e2e/k8s/scripts/install_helm_e2e.sh; then

docs/set-up/config-reference.mdx

@@ -268,8 +268,8 @@ jobs:
       storage:
         # Name of the Docker volume for persistent storage | default: 'nemo-jobs-storage'
         volume_name: nemo-jobs-storage
-        # Docker image used to set permissions on the volume | default: 'busybox'
-        volume_permissions_image: busybox
+        # Docker image used to set permissions on the volume | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # List of additional Docker volume mounts for the job
         additional_volume_mounts: []
       # Docker networking configuration
@@ -358,8 +358,8 @@ jobs:
       storage:
         # Persistent Volume Claim Name to use for job storage. | default: ''
         pvc_name: ''
-        # Image used to set volume permissions | default: 'busybox'
-        volume_permissions_image: busybox
+        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # Additional volumes to mount
         additional_volumes: []
         # Additional volume mounts
@@ -436,8 +436,8 @@ jobs:
       storage:
         # Persistent Volume Claim Name to use for job storage. | default: ''
         pvc_name: ''
-        # Image used to set volume permissions | default: 'busybox'
-        volume_permissions_image: busybox
+        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # Additional volumes to mount
         additional_volumes: []
         # Additional volume mounts

e2e/k8s/scripts/install_helm_e2e.sh

@@ -22,7 +22,7 @@ NMP_E2E_TAG="${NMP_E2E_TAG:-}"
 NMP_E2E_PULL_POLICY="${NMP_E2E_PULL_POLICY:-}"
 REQUIRE_NMP_E2E_IMAGES="${REQUIRE_NMP_E2E_IMAGES:-false}"
 POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}"
-BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox}"
+BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox:stable}"
 RELEASE_READY_SCRIPT="${RELEASE_READY_SCRIPT:-${SCRIPT_DIR}/wait_for_release_ready.sh}"
 INSTALL_RUSTFS="${INSTALL_RUSTFS:-false}"
 RUSTFS_STORAGECLASS="${RUSTFS_STORAGECLASS:-standard}"

e2e/k8s/scripts/install_nmp_auth_e2e.sh

@@ -12,7 +12,7 @@ export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube-auth.yam
 export NMP_E2E_REGISTRY="${NMP_E2E_REGISTRY:-my-registry}"
 export NMP_E2E_TAG="${NMP_E2E_TAG:-local}"
 export POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}"
-export BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-busybox}"
+export BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox:stable}"
 
 export REQUIRE_NMP_E2E_IMAGES="${REQUIRE_NMP_E2E_IMAGES:-true}"
 

e2e/k8s/values/default.yaml

@@ -35,4 +35,4 @@ platformConfig:
 core:
   storage:
     storageClass: oci-nfs
-    volumePermissionsImage: docker.io/library/busybox
+    volumePermissionsImage: docker.io/library/busybox:stable

e2e/k8s/values/kind.yaml

@@ -11,7 +11,7 @@ core:
   storage:
     accessModes:
       - ReadWriteOnce
-    volumePermissionsImage: docker.io/library/busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: false

e2e/k8s/values/minikube-auth-portforward.yaml

@@ -15,7 +15,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-    volumePermissionsImage: busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 envoyProxy:
   enabled: false

e2e/k8s/values/minikube-auth.yaml

@@ -17,7 +17,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-    volumePermissionsImage: busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: true

e2e/k8s/values/minikube.yaml

@@ -12,7 +12,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-  volumePermissionsImage: docker.io/library/busybox
+  volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: true

k8s/helm/README.md

@@ -154,7 +154,7 @@ secrets will not decrypt with a new key.
 | core.storage.existingPersistentVolumeName | string | `""` | If set, pods will mount this persistent volume for job-scoped storage and we will not create a new persistent volume claim. |
 | core.storage.size | string | `"200Gi"` | size of the persistent volume claim used for persistent storage |
 | core.storage.storageClass | string | `""` | Which storageClass to use when creating a new persistent volume claim. Empty string uses the cluster's default StorageClass. |
-| core.storage.volumePermissionsImage | string | `"docker.io/library/busybox"` | volumePermissionsImage is the image used to set permissions on the volume |
+| core.storage.volumePermissionsImage | string | `"docker.io/library/busybox:stable"` | volumePermissionsImage is the image used to set permissions on the volume |
 | core.telemetry | object | `{}` | OpenTelemetry configuration overrides for the platform deployment. |
 | env | object | `{}` | Environment variables that will be applied to every deployment pod. Uses a simple key value map structure like MY_ENV_VAR: the-key and works with valueFrom as well. |
 | envFromSecret | string | `""` | Optional. Name of an existing Kubernetes Secret to load as env vars (envFrom) for the API pod. When set, the chart does not create or generate the default api-env Secret; use your own Secret (for example, from Vault or sealed-secrets). |

k8s/helm/values.yaml

@@ -720,7 +720,7 @@ core:
     # -- size of the persistent volume claim used for persistent storage
     size: 200Gi
     # -- volumePermissionsImage is the image used to set permissions on the volume
-    volumePermissionsImage: "docker.io/library/busybox"
+    volumePermissionsImage: "docker.io/library/busybox:stable"
     # -- Annotations to add to the persistent volume claim
     annotations: {}
 

packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/execution_profiles.py

@@ -39,7 +39,7 @@
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 # Default image used to set filesystem permissions on job storage volumes.
-DEFAULT_VOLUME_PERMISSIONS_IMAGE = "busybox"
+DEFAULT_VOLUME_PERMISSIONS_IMAGE = "docker.io/library/busybox:stable"
 JOB_LOGS_ENDPOINT_ENVVAR = "NMP_JOB_LOGS_ENDPOINT"
 
 # Env var names set by the platform during job creation; user-provided profile

packages/nmp_customization_common/src/nmp/customization_common/training/nccl.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""NCCL environment helpers shared by customization training backends."""
+
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_IB_SYSFS = Path("/sys/class/infiniband")
+
+
+def maybe_set_nccl_ib_hca() -> None:
+    """Set NCCL_IB_HCA to usable Mellanox HCAs when phantom devices are present.
+
+    Some hosts expose mlx InfiniBand devices in sysfs that have no netdev. NCCL
+    will try those and fail; filter them out when that happens.
+    """
+    if os.environ.get("NCCL_IB_HCA") or not _IB_SYSFS.is_dir():
+        return
+
+    usable: list[str] = []
+    phantom: list[str] = []
+    try:
+        hcas = sorted(p for p in _IB_SYSFS.iterdir() if p.is_dir())
+    except OSError:
+        return
+
+    for hca in hcas:
+        if not hca.name.startswith("mlx"):
+            continue
+        try:
+            has_netdev = any((hca / "device" / "net").iterdir())
+        except OSError:
+            has_netdev = False
+        (usable if has_netdev else phantom).append(hca.name)
+
+    if not usable or not phantom:
+        return
+
+    os.environ["NCCL_IB_HCA"] = ",".join(usable)
+    logger.info("Setting NCCL_IB_HCA=%s (excluded phantom HCAs: %s)", os.environ["NCCL_IB_HCA"], ",".join(phantom))

packages/nmp_customization_common/tests/training/test_nccl.py

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from pathlib import Path
+
+import pytest
+from nmp.customization_common.training import nccl as nccl_mod
+from nmp.customization_common.training.nccl import maybe_set_nccl_ib_hca
+
+
+def _make_hca(ib_root: Path, name: str, with_netdev: bool = False) -> None:
+    device = ib_root / name / "device"
+    net = device / "net"
+    net.mkdir(parents=True)
+    if with_netdev:
+        (net / "eth0").mkdir()
+
+
+@pytest.fixture
+def ib_sysfs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    ib = tmp_path / "infiniband"
+    ib.mkdir()
+    monkeypatch.setattr(nccl_mod, "_IB_SYSFS", ib)
+    monkeypatch.delenv("NCCL_IB_HCA", raising=False)
+    return ib
+
+
+def test_sets_nccl_ib_hca_when_phantoms_exist(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_0", with_netdev=True)
+    _make_hca(ib_sysfs, "mlx5_1", with_netdev=False)
+
+    maybe_set_nccl_ib_hca()
+    assert os.environ["NCCL_IB_HCA"] == "mlx5_0"
+
+
+def test_noop_without_phantoms(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_0", with_netdev=True)
+
+    maybe_set_nccl_ib_hca()
+    assert "NCCL_IB_HCA" not in os.environ
+
+
+def test_noop_without_ib(ib_sysfs: Path) -> None:
+    ib_sysfs.rmdir()
+
+    maybe_set_nccl_ib_hca()
+    assert "NCCL_IB_HCA" not in os.environ
+
+
+def test_respects_existing_value(ib_sysfs: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    _make_hca(ib_sysfs, "mlx5_0", with_netdev=True)
+    _make_hca(ib_sysfs, "mlx5_1", with_netdev=False)
+    monkeypatch.setenv("NCCL_IB_HCA", "mlx5_custom")
+
+    maybe_set_nccl_ib_hca()
+    assert os.environ["NCCL_IB_HCA"] == "mlx5_custom"

plugins/nemo-automodel/src/nemo_automodel_plugin/jobs/jobs.py

@@ -14,11 +14,12 @@
 from typing import ClassVar, cast
 
 from nemo_automodel_plugin.config import get_config
-from nemo_automodel_plugin.schema import AutomodelJobInput, AutomodelJobOutput
+from nemo_automodel_plugin.schema import AutomodelJobInput, AutomodelJobOutput, ValidationError
 from nemo_automodel_plugin.transform import transform_input_to_output
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec
 from nemo_platform_plugin.jobs.docker import validate_gpu_available_for_docker
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
 from nmp.automodel.compile import platform_job_config_compiler
 from nmp.customization_common.contributor.jobs import BaseSubmitJob, require_container_runtime
 from pydantic import BaseModel
@@ -56,7 +57,10 @@ async def compile(
         # Multi-node jobs compile to a gpu_distributed (Volcano) executor, which
         # only exists on Kubernetes; gate here so docker platforms fail fast.
         require_container_runtime(cls.runtime_label, num_nodes=canonical.parallelism.num_nodes)
-        canonical.validate_for_training()
+        try:
+            canonical.validate_for_training()
+        except ValidationError as e:
+            raise PlatformJobCompilationError(str(e)) from e
 
         plugin_config = get_config()
         execution_profile = (

plugins/nemo-automodel/tests/test_jobs.py

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for AutomodelJob.compile error mapping.
+
+``validate_for_training`` rejects inconsistent parallelism/batch topologies.
+Those are user input errors, so they have to leave ``compile`` as
+``PlatformJobCompilationError`` — the api_factory only maps that type to a
+422, and anything else escapes as a 500.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+from nemo_automodel_plugin.jobs.jobs import AutomodelJob
+from nemo_automodel_plugin.schema import AutomodelJobOutput
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
+
+
+def _make_canonical(**parallelism: Any) -> AutomodelJobOutput:
+    return AutomodelJobOutput.model_validate(
+        {
+            "model": "default/base",
+            "dataset": {"training": "default/train"},
+            "training": {"training_type": "sft"},
+            "schedule": {"epochs": 1},
+            "batch": {"global_batch_size": 4, "micro_batch_size": 1},
+            "optimizer": {},
+            "parallelism": {"num_nodes": 2, "num_gpus_per_node": 8, **parallelism},
+            "output": {"name": "out", "type": "adapter", "fileset": "out-fs"},
+        },
+    )
+
+
+def _compile(canonical: AutomodelJobOutput) -> Any:
+    return asyncio.run(
+        AutomodelJob.compile(
+            workspace="default",
+            spec=canonical,
+            entity_client=object(),
+            job_name=None,
+            async_sdk=object(),
+        ),
+    )
+
+
+class TestCompileValidationErrors:
+    def test_indivisible_batch_raises_compilation_error(self) -> None:
+        # global_batch_size=4 with data_parallel_size=16 (2 nodes x 8 GPUs, TP=1).
+        canonical = _make_canonical(tensor_parallel_size=1)
+        with patch("nemo_automodel_plugin.jobs.jobs.require_container_runtime"):
+            with pytest.raises(PlatformJobCompilationError, match="global_batch_size"):
+                _compile(canonical)
+
+    def test_indivisible_model_parallel_raises_compilation_error(self) -> None:
+        # 16 total GPUs is not divisible by tensor_parallel_size=5.
+        canonical = _make_canonical(tensor_parallel_size=5)
+        with patch("nemo_automodel_plugin.jobs.jobs.require_container_runtime"):
+            with pytest.raises(PlatformJobCompilationError, match="Total GPUs"):
+                _compile(canonical)

plugins/nemo-rl/src/nemo_rl_plugin/jobs/jobs.py

@@ -19,6 +19,7 @@
 
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
 from nemo_rl_plugin.schema import RlJobInput
 from nemo_rl_plugin.transform import transform_input_to_output
 from nmp.customization_common.contributor.jobs import BaseSubmitJob, require_distributed_runtime
@@ -63,7 +64,10 @@ async def compile(
         del entity_client, options
         require_distributed_runtime(cls.runtime_label)
         canonical = spec if isinstance(spec, RlJobOutput) else RlJobOutput.model_validate(spec.model_dump())
-        canonical.validate_for_training()
+        try:
+            canonical.validate_for_training()
+        except ValueError as e:
+            raise PlatformJobCompilationError(str(e)) from e
 
         # Leave ``None`` when unset so the compiler can default per topology.
         execution_profile = canonical.training.execution_profile or profile