fix(customizer): fix 500 on validation failure for automodel, busybox default

.github/actions/setup-kind-cluster/action.yaml

@@ -163,7 +163,7 @@ runs:
         HELM_VALUES: ${{ inputs['helm-values'] }}
         REQUIRE_NMP_E2E_IMAGES: "true"
         POSTGRES_IMAGE: docker.io/library/postgres
-        BUSYBOX_IMAGE: docker.io/library/busybox
+        BUSYBOX_IMAGE: docker.io/library/busybox:stable
         GITHUB_TOKEN: ${{ inputs['kind-image-pull-token'] }}
       run: |
         if ! e2e/k8s/scripts/install_helm_e2e.sh; then

docs/set-up/config-reference.mdx

@@ -268,8 +268,8 @@ jobs:
       storage:
         # Name of the Docker volume for persistent storage | default: 'nemo-jobs-storage'
         volume_name: nemo-jobs-storage
-        # Docker image used to set permissions on the volume | default: 'busybox'
-        volume_permissions_image: busybox
+        # Docker image used to set permissions on the volume | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # List of additional Docker volume mounts for the job
         additional_volume_mounts: []
       # Docker networking configuration
@@ -358,8 +358,8 @@ jobs:
       storage:
         # Persistent Volume Claim Name to use for job storage. | default: ''
         pvc_name: ''
-        # Image used to set volume permissions | default: 'busybox'
-        volume_permissions_image: busybox
+        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # Additional volumes to mount
         additional_volumes: []
         # Additional volume mounts
@@ -436,8 +436,8 @@ jobs:
       storage:
         # Persistent Volume Claim Name to use for job storage. | default: ''
         pvc_name: ''
-        # Image used to set volume permissions | default: 'busybox'
-        volume_permissions_image: busybox
+        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # Additional volumes to mount
         additional_volumes: []
         # Additional volume mounts

e2e/k8s/scripts/install_helm_e2e.sh

@@ -22,7 +22,7 @@ NMP_E2E_TAG="${NMP_E2E_TAG:-}"
 NMP_E2E_PULL_POLICY="${NMP_E2E_PULL_POLICY:-}"
 REQUIRE_NMP_E2E_IMAGES="${REQUIRE_NMP_E2E_IMAGES:-false}"
 POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}"
-BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox}"
+BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox:stable}"
 RELEASE_READY_SCRIPT="${RELEASE_READY_SCRIPT:-${SCRIPT_DIR}/wait_for_release_ready.sh}"
 INSTALL_RUSTFS="${INSTALL_RUSTFS:-false}"
 RUSTFS_STORAGECLASS="${RUSTFS_STORAGECLASS:-standard}"

e2e/k8s/scripts/install_nmp_auth_e2e.sh

@@ -12,7 +12,7 @@ export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube-auth.yam
 export NMP_E2E_REGISTRY="${NMP_E2E_REGISTRY:-my-registry}"
 export NMP_E2E_TAG="${NMP_E2E_TAG:-local}"
 export POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}"
-export BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-busybox}"
+export BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox:stable}"
 
 export REQUIRE_NMP_E2E_IMAGES="${REQUIRE_NMP_E2E_IMAGES:-true}"
 

e2e/k8s/values/default.yaml

@@ -35,4 +35,4 @@ platformConfig:
 core:
   storage:
     storageClass: oci-nfs
-    volumePermissionsImage: docker.io/library/busybox
+    volumePermissionsImage: docker.io/library/busybox:stable

e2e/k8s/values/kind.yaml

@@ -11,7 +11,7 @@ core:
   storage:
     accessModes:
       - ReadWriteOnce
-    volumePermissionsImage: docker.io/library/busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: false

e2e/k8s/values/minikube-auth-portforward.yaml

@@ -15,7 +15,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-    volumePermissionsImage: busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 envoyProxy:
   enabled: false

e2e/k8s/values/minikube-auth.yaml

@@ -17,7 +17,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-    volumePermissionsImage: busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: true

e2e/k8s/values/minikube.yaml

@@ -12,7 +12,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-  volumePermissionsImage: docker.io/library/busybox
+  volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: true

k8s/helm/README.md

@@ -154,7 +154,7 @@ secrets will not decrypt with a new key.
 | core.storage.existingPersistentVolumeName | string | `""` | If set, pods will mount this persistent volume for job-scoped storage and we will not create a new persistent volume claim. |
 | core.storage.size | string | `"200Gi"` | size of the persistent volume claim used for persistent storage |
 | core.storage.storageClass | string | `""` | Which storageClass to use when creating a new persistent volume claim. Empty string uses the cluster's default StorageClass. |
-| core.storage.volumePermissionsImage | string | `"docker.io/library/busybox"` | volumePermissionsImage is the image used to set permissions on the volume |
+| core.storage.volumePermissionsImage | string | `"docker.io/library/busybox:stable"` | volumePermissionsImage is the image used to set permissions on the volume |
 | core.telemetry | object | `{}` | OpenTelemetry configuration overrides for the platform deployment. |
 | env | object | `{}` | Environment variables that will be applied to every deployment pod. Uses a simple key value map structure like MY_ENV_VAR: the-key and works with valueFrom as well. |
 | envFromSecret | string | `""` | Optional. Name of an existing Kubernetes Secret to load as env vars (envFrom) for the API pod. When set, the chart does not create or generate the default api-env Secret; use your own Secret (for example, from Vault or sealed-secrets). |

k8s/helm/values.yaml

@@ -720,7 +720,7 @@ core:
     # -- size of the persistent volume claim used for persistent storage
     size: 200Gi
     # -- volumePermissionsImage is the image used to set permissions on the volume
-    volumePermissionsImage: "docker.io/library/busybox"
+    volumePermissionsImage: "docker.io/library/busybox:stable"
     # -- Annotations to add to the persistent volume claim
     annotations: {}
 

packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/execution_profiles.py

@@ -39,7 +39,7 @@
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 # Default image used to set filesystem permissions on job storage volumes.
-DEFAULT_VOLUME_PERMISSIONS_IMAGE = "busybox"
+DEFAULT_VOLUME_PERMISSIONS_IMAGE = "docker.io/library/busybox:stable"
 JOB_LOGS_ENDPOINT_ENVVAR = "NMP_JOB_LOGS_ENDPOINT"
 
 # Env var names set by the platform during job creation; user-provided profile

packages/nmp_customization_common/src/nmp/customization_common/training/nccl.py

@@ -0,0 +1,48 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""NCCL environment helpers shared by customization training backends."""
+
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_IB_SYSFS = Path("/sys/class/infiniband")
+
+
+def get_nccl_ib_env() -> dict[str, str]:
+    """Return NCCL overrides when Mellanox HCAs lack network devices."""
+    if os.environ.get("NCCL_IB_HCA") or os.environ.get("NCCL_IB_DISABLE") or not _IB_SYSFS.is_dir():
+        return {}
+
+    usable: list[str] = []
+    phantom: list[str] = []
+    try:
+        hcas = sorted(path for path in _IB_SYSFS.iterdir() if path.is_dir())
+    except OSError:
+        return {}
+
+    for hca in hcas:
+        if not hca.name.startswith("mlx"):
+            continue
+        try:
+            has_netdev = any((hca / "device" / "net").iterdir())
+        except OSError:
+            has_netdev = False
+        if has_netdev:
+            usable.append(hca.name)
+        else:
+            phantom.append(hca.name)
+
+    if not phantom:
+        return {}
+
+    if not usable:
+        logger.info("Disabling NCCL IB because all detected Mellanox HCAs lack network devices")
+        return {"NCCL_IB_DISABLE": "1"}
+
+    hca_filter = ",".join(f"={hca}" for hca in usable)
+    logger.info("Setting NCCL_IB_HCA=%s (excluded phantom HCAs: %s)", hca_filter, ",".join(phantom))
+    return {"NCCL_IB_HCA": hca_filter}

packages/nmp_customization_common/tests/test_nccl.py

@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from pathlib import Path
+
+import pytest
+from nmp.customization_common.training import nccl as nccl_module
+from nmp.customization_common.training.nccl import get_nccl_ib_env
+
+
+def _make_hca(ib_root: Path, name: str, with_netdev: bool = False) -> None:
+    net = ib_root / name / "device" / "net"
+    net.mkdir(parents=True)
+    if with_netdev:
+        (net / "eth0").mkdir()
+
+
+@pytest.fixture
+def ib_sysfs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    ib = tmp_path / "infiniband"
+    ib.mkdir()
+    monkeypatch.setattr(nccl_module, "_IB_SYSFS", ib)
+    monkeypatch.delenv("NCCL_IB_HCA", raising=False)
+    monkeypatch.delenv("NCCL_IB_DISABLE", raising=False)
+    return ib
+
+
+def test_returns_exact_usable_hcas_when_phantoms_exist(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_1", with_netdev=True)
+    _make_hca(ib_sysfs, "mlx5_10", with_netdev=True)
+    _make_hca(ib_sysfs, "mlx5_2")
+
+    assert get_nccl_ib_env() == {"NCCL_IB_HCA": "=mlx5_1,=mlx5_10"}
+    assert "NCCL_IB_HCA" not in os.environ
+
+
+def test_disables_ib_when_all_hcas_are_phantom(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_0")
+    _make_hca(ib_sysfs, "mlx5_1")
+
+    assert get_nccl_ib_env() == {"NCCL_IB_DISABLE": "1"}
+
+
+def test_returns_no_overrides_without_phantoms(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_0", with_netdev=True)
+
+    assert get_nccl_ib_env() == {}
+
+
+def test_respects_existing_nccl_configuration(ib_sysfs: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    _make_hca(ib_sysfs, "mlx5_0")
+    monkeypatch.setenv("NCCL_IB_HCA", "mlx5_custom")
+
+    assert get_nccl_ib_env() == {}

packages/nmp_customization_common/tests/training/test_nccl.py

@@ -0,0 +1,57 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from pathlib import Path
+
+import pytest
+from nmp.customization_common.training import nccl as nccl_mod
+from nmp.customization_common.training.nccl import maybe_set_nccl_ib_hca
+
+
+def _make_hca(ib_root: Path, name: str, with_netdev: bool = False) -> None:
+    device = ib_root / name / "device"
+    net = device / "net"
+    net.mkdir(parents=True)
+    if with_netdev:
+        (net / "eth0").mkdir()
+
+
+@pytest.fixture
+def ib_sysfs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    ib = tmp_path / "infiniband"
+    ib.mkdir()
+    monkeypatch.setattr(nccl_mod, "_IB_SYSFS", ib)
+    monkeypatch.delenv("NCCL_IB_HCA", raising=False)
+    return ib
+
+
+def test_sets_nccl_ib_hca_when_phantoms_exist(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_0", with_netdev=True)
+    _make_hca(ib_sysfs, "mlx5_1", with_netdev=False)
+
+    maybe_set_nccl_ib_hca()
+    assert os.environ["NCCL_IB_HCA"] == "mlx5_0"
+
+
+def test_noop_without_phantoms(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_0", with_netdev=True)
+
+    maybe_set_nccl_ib_hca()
+    assert "NCCL_IB_HCA" not in os.environ
+
+
+def test_noop_without_ib(ib_sysfs: Path) -> None:
+    ib_sysfs.rmdir()
+
+    maybe_set_nccl_ib_hca()
+    assert "NCCL_IB_HCA" not in os.environ
+
+
+def test_respects_existing_value(ib_sysfs: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    _make_hca(ib_sysfs, "mlx5_0", with_netdev=True)
+    _make_hca(ib_sysfs, "mlx5_1", with_netdev=False)
+    monkeypatch.setenv("NCCL_IB_HCA", "mlx5_custom")
+
+    maybe_set_nccl_ib_hca()
+    assert os.environ["NCCL_IB_HCA"] == "mlx5_custom"

plugins/nemo-automodel/src/nemo_automodel_plugin/jobs/jobs.py

@@ -14,11 +14,12 @@
 from typing import ClassVar, cast
 
 from nemo_automodel_plugin.config import get_config
-from nemo_automodel_plugin.schema import AutomodelJobInput, AutomodelJobOutput
+from nemo_automodel_plugin.schema import AutomodelJobInput, AutomodelJobOutput, ValidationError
 from nemo_automodel_plugin.transform import transform_input_to_output
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec
 from nemo_platform_plugin.jobs.docker import validate_gpu_available_for_docker
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
 from nmp.automodel.compile import platform_job_config_compiler
 from nmp.customization_common.contributor.jobs import BaseSubmitJob, require_container_runtime
 from pydantic import BaseModel
@@ -56,7 +57,10 @@ async def compile(
         # Multi-node jobs compile to a gpu_distributed (Volcano) executor, which
         # only exists on Kubernetes; gate here so docker platforms fail fast.
         require_container_runtime(cls.runtime_label, num_nodes=canonical.parallelism.num_nodes)
-        canonical.validate_for_training()
+        try:
+            canonical.validate_for_training()
+        except ValidationError as e:
+            raise PlatformJobCompilationError(str(e)) from e
 
         plugin_config = get_config()
         execution_profile = (