diff --git a/.github/actions/setup-kind-cluster/action.yaml b/.github/actions/setup-kind-cluster/action.yaml
index 398f51f818..3c9b207e01 100644
--- a/.github/actions/setup-kind-cluster/action.yaml
+++ b/.github/actions/setup-kind-cluster/action.yaml
@@ -163,7 +163,7 @@ runs:
         HELM_VALUES: ${{ inputs['helm-values'] }}
         REQUIRE_NMP_E2E_IMAGES: "true"
         POSTGRES_IMAGE: docker.io/library/postgres
-        BUSYBOX_IMAGE: docker.io/library/busybox
+        BUSYBOX_IMAGE: docker.io/library/busybox:stable
         GITHUB_TOKEN: ${{ inputs['kind-image-pull-token'] }}
       run: |
         if ! e2e/k8s/scripts/install_helm_e2e.sh; then
diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx
index f602736ddb..cf2c95eb95 100644
--- a/docs/set-up/config-reference.mdx
+++ b/docs/set-up/config-reference.mdx
@@ -268,8 +268,8 @@ jobs:
       storage:
         # Name of the Docker volume for persistent storage | default: 'nemo-jobs-storage'
         volume_name: nemo-jobs-storage
-        # Docker image used to set permissions on the volume | default: 'busybox'
-        volume_permissions_image: busybox
+        # Docker image used to set permissions on the volume | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # List of additional Docker volume mounts for the job
         additional_volume_mounts: []
       # Docker networking configuration
@@ -358,8 +358,8 @@ jobs:
       storage:
         # Persistent Volume Claim Name to use for job storage. | default: ''
         pvc_name: ''
-        # Image used to set volume permissions | default: 'busybox'
-        volume_permissions_image: busybox
+        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # Additional volumes to mount
         additional_volumes: []
         # Additional volume mounts
@@ -436,8 +436,8 @@ jobs:
       storage:
         # Persistent Volume Claim Name to use for job storage. | default: ''
         pvc_name: ''
-        # Image used to set volume permissions | default: 'busybox'
-        volume_permissions_image: busybox
+        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # Additional volumes to mount
         additional_volumes: []
         # Additional volume mounts
diff --git a/e2e/k8s/scripts/install_helm_e2e.sh b/e2e/k8s/scripts/install_helm_e2e.sh
index 5cdd785051..6d4899c56b 100755
--- a/e2e/k8s/scripts/install_helm_e2e.sh
+++ b/e2e/k8s/scripts/install_helm_e2e.sh
@@ -22,7 +22,7 @@ NMP_E2E_TAG="${NMP_E2E_TAG:-}"
 NMP_E2E_PULL_POLICY="${NMP_E2E_PULL_POLICY:-}"
 REQUIRE_NMP_E2E_IMAGES="${REQUIRE_NMP_E2E_IMAGES:-false}"
 POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}"
-BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox}"
+BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox:stable}"
 RELEASE_READY_SCRIPT="${RELEASE_READY_SCRIPT:-${SCRIPT_DIR}/wait_for_release_ready.sh}"
 INSTALL_RUSTFS="${INSTALL_RUSTFS:-false}"
 RUSTFS_STORAGECLASS="${RUSTFS_STORAGECLASS:-standard}"
diff --git a/e2e/k8s/scripts/install_nmp_auth_e2e.sh b/e2e/k8s/scripts/install_nmp_auth_e2e.sh
index ceff9c1488..fe25de79ac 100755
--- a/e2e/k8s/scripts/install_nmp_auth_e2e.sh
+++ b/e2e/k8s/scripts/install_nmp_auth_e2e.sh
@@ -12,7 +12,7 @@ export HELM_VALUES="${HELM_VALUES:-${REPO_ROOT}/e2e/k8s/values/minikube-auth.yam
 export NMP_E2E_REGISTRY="${NMP_E2E_REGISTRY:-my-registry}"
 export NMP_E2E_TAG="${NMP_E2E_TAG:-local}"
 export POSTGRES_IMAGE="${POSTGRES_IMAGE:-docker.io/library/postgres}"
-export BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-busybox}"
+export BUSYBOX_IMAGE="${BUSYBOX_IMAGE:-docker.io/library/busybox:stable}"
 
 export REQUIRE_NMP_E2E_IMAGES="${REQUIRE_NMP_E2E_IMAGES:-true}"
 
diff --git a/e2e/k8s/values/default.yaml b/e2e/k8s/values/default.yaml
index e10e8bdb08..b8ece2c62b 100644
--- a/e2e/k8s/values/default.yaml
+++ b/e2e/k8s/values/default.yaml
@@ -35,4 +35,4 @@ platformConfig:
 core:
   storage:
     storageClass: oci-nfs
-    volumePermissionsImage: docker.io/library/busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
diff --git a/e2e/k8s/values/kind.yaml b/e2e/k8s/values/kind.yaml
index ecc99ce2c3..c22548b34a 100644
--- a/e2e/k8s/values/kind.yaml
+++ b/e2e/k8s/values/kind.yaml
@@ -11,7 +11,7 @@ core:
   storage:
     accessModes:
       - ReadWriteOnce
-    volumePermissionsImage: docker.io/library/busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: false
diff --git a/e2e/k8s/values/minikube-auth-portforward.yaml b/e2e/k8s/values/minikube-auth-portforward.yaml
index 28cd5398cb..11930654ea 100644
--- a/e2e/k8s/values/minikube-auth-portforward.yaml
+++ b/e2e/k8s/values/minikube-auth-portforward.yaml
@@ -15,7 +15,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-    volumePermissionsImage: busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 envoyProxy:
   enabled: false
diff --git a/e2e/k8s/values/minikube-auth.yaml b/e2e/k8s/values/minikube-auth.yaml
index 798997dc08..c7e179fe83 100644
--- a/e2e/k8s/values/minikube-auth.yaml
+++ b/e2e/k8s/values/minikube-auth.yaml
@@ -17,7 +17,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-    volumePermissionsImage: busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: true
diff --git a/e2e/k8s/values/minikube.yaml b/e2e/k8s/values/minikube.yaml
index 33c97acc1f..a55c75a221 100644
--- a/e2e/k8s/values/minikube.yaml
+++ b/e2e/k8s/values/minikube.yaml
@@ -12,7 +12,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-  volumePermissionsImage: docker.io/library/busybox
+  volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: true
diff --git a/k8s/helm/README.md b/k8s/helm/README.md
index 56ed5ee3f7..24420ead41 100644
--- a/k8s/helm/README.md
+++ b/k8s/helm/README.md
@@ -154,7 +154,7 @@ secrets will not decrypt with a new key.
 | core.storage.existingPersistentVolumeName | string | `""` | If set, pods will mount this persistent volume for job-scoped storage and we will not create a new persistent volume claim. |
 | core.storage.size | string | `"200Gi"` | size of the persistent volume claim used for persistent storage |
 | core.storage.storageClass | string | `""` | Which storageClass to use when creating a new persistent volume claim. Empty string uses the cluster's default StorageClass. |
-| core.storage.volumePermissionsImage | string | `"docker.io/library/busybox"` | volumePermissionsImage is the image used to set permissions on the volume |
+| core.storage.volumePermissionsImage | string | `"docker.io/library/busybox:stable"` | volumePermissionsImage is the image used to set permissions on the volume |
 | core.telemetry | object | `{}` | OpenTelemetry configuration overrides for the platform deployment. |
 | env | object | `{}` | Environment variables that will be applied to every deployment pod. Uses a simple key value map structure like MY_ENV_VAR: the-key and works with valueFrom as well. |
 | envFromSecret | string | `""` | Optional. Name of an existing Kubernetes Secret to load as env vars (envFrom) for the API pod. When set, the chart does not create or generate the default api-env Secret; use your own Secret (for example, from Vault or sealed-secrets). |
diff --git a/k8s/helm/values.yaml b/k8s/helm/values.yaml
index 361a9f06c1..555ca06d97 100644
--- a/k8s/helm/values.yaml
+++ b/k8s/helm/values.yaml
@@ -720,7 +720,7 @@ core:
     # -- size of the persistent volume claim used for persistent storage
     size: 200Gi
     # -- volumePermissionsImage is the image used to set permissions on the volume
-    volumePermissionsImage: "docker.io/library/busybox"
+    volumePermissionsImage: "docker.io/library/busybox:stable"
     # -- Annotations to add to the persistent volume claim
     annotations: {}
 
diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml
index 829a0150ee..f07d6e4b4d 100644
--- a/openapi/ga/individual/platform.openapi.yaml
+++ b/openapi/ga/individual/platform.openapi.yaml
@@ -10172,7 +10172,7 @@ components:
           type: string
           title: Volume Permissions Image
           description: Docker image used to set permissions on the volume
-          default: busybox
+          default: docker.io/library/busybox:stable
         additional_volume_mounts:
           items:
             $ref: '#/components/schemas/DockerVolumeMount'
@@ -13177,7 +13177,7 @@ components:
           type: string
           title: Volume Permissions Image
           description: Image used to set volume permissions
-          default: busybox
+          default: docker.io/library/busybox:stable
         additional_volumes:
           items:
             $ref: '#/components/schemas/KubernetesVolume'
diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml
index 829a0150ee..f07d6e4b4d 100644
--- a/openapi/ga/openapi.yaml
+++ b/openapi/ga/openapi.yaml
@@ -10172,7 +10172,7 @@ components:
           type: string
           title: Volume Permissions Image
           description: Docker image used to set permissions on the volume
-          default: busybox
+          default: docker.io/library/busybox:stable
         additional_volume_mounts:
           items:
             $ref: '#/components/schemas/DockerVolumeMount'
@@ -13177,7 +13177,7 @@ components:
           type: string
           title: Volume Permissions Image
           description: Image used to set volume permissions
-          default: busybox
+          default: docker.io/library/busybox:stable
         additional_volumes:
           items:
             $ref: '#/components/schemas/KubernetesVolume'
diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml
index 829a0150ee..f07d6e4b4d 100644
--- a/openapi/openapi.yaml
+++ b/openapi/openapi.yaml
@@ -10172,7 +10172,7 @@ components:
           type: string
           title: Volume Permissions Image
           description: Docker image used to set permissions on the volume
-          default: busybox
+          default: docker.io/library/busybox:stable
         additional_volume_mounts:
           items:
             $ref: '#/components/schemas/DockerVolumeMount'
@@ -13177,7 +13177,7 @@ components:
           type: string
           title: Volume Permissions Image
           description: Image used to set volume permissions
-          default: busybox
+          default: docker.io/library/busybox:stable
         additional_volumes:
           items:
             $ref: '#/components/schemas/KubernetesVolume'
diff --git a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/execution_profiles.py b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/execution_profiles.py
index 20c9b17025..fc803cc8e6 100644
--- a/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/execution_profiles.py
+++ b/packages/nemo_platform_plugin/src/nemo_platform_plugin/jobs/execution_profiles.py
@@ -39,7 +39,7 @@
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 # Default image used to set filesystem permissions on job storage volumes.
-DEFAULT_VOLUME_PERMISSIONS_IMAGE = "busybox"
+DEFAULT_VOLUME_PERMISSIONS_IMAGE = "docker.io/library/busybox:stable"
 JOB_LOGS_ENDPOINT_ENVVAR = "NMP_JOB_LOGS_ENDPOINT"
 
 # Env var names set by the platform during job creation; user-provided profile
diff --git a/packages/nmp_customization_common/src/nmp/customization_common/training/nccl.py b/packages/nmp_customization_common/src/nmp/customization_common/training/nccl.py
new file mode 100644
index 0000000000..bde0912b8e
--- /dev/null
+++ b/packages/nmp_customization_common/src/nmp/customization_common/training/nccl.py
@@ -0,0 +1,48 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""NCCL environment helpers shared by customization training backends."""
+
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger(__name__)
+
+_IB_SYSFS = Path("/sys/class/infiniband")
+
+
+def get_nccl_ib_env() -> dict[str, str]:
+    """Return NCCL overrides when Mellanox HCAs lack network devices."""
+    if os.environ.get("NCCL_IB_HCA") or os.environ.get("NCCL_IB_DISABLE") or not _IB_SYSFS.is_dir():
+        return {}
+
+    usable: list[str] = []
+    phantom: list[str] = []
+    try:
+        hcas = sorted(path for path in _IB_SYSFS.iterdir() if path.is_dir())
+    except OSError:
+        return {}
+
+    for hca in hcas:
+        if not hca.name.startswith("mlx"):
+            continue
+        try:
+            has_netdev = any((hca / "device" / "net").iterdir())
+        except OSError:
+            has_netdev = False
+        if has_netdev:
+            usable.append(hca.name)
+        else:
+            phantom.append(hca.name)
+
+    if not phantom:
+        return {}
+
+    if not usable:
+        logger.info("Disabling NCCL IB because all detected Mellanox HCAs lack network devices")
+        return {"NCCL_IB_DISABLE": "1"}
+
+    hca_filter = ",".join(f"={hca}" for hca in usable)
+    logger.info("Setting NCCL_IB_HCA=%s (excluded phantom HCAs: %s)", hca_filter, ",".join(phantom))
+    return {"NCCL_IB_HCA": hca_filter}
diff --git a/packages/nmp_customization_common/tests/test_nccl.py b/packages/nmp_customization_common/tests/test_nccl.py
new file mode 100644
index 0000000000..855432e2db
--- /dev/null
+++ b/packages/nmp_customization_common/tests/test_nccl.py
@@ -0,0 +1,55 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+import os
+from pathlib import Path
+
+import pytest
+from nmp.customization_common.training import nccl as nccl_module
+from nmp.customization_common.training.nccl import get_nccl_ib_env
+
+
+def _make_hca(ib_root: Path, name: str, with_netdev: bool = False) -> None:
+    net = ib_root / name / "device" / "net"
+    net.mkdir(parents=True)
+    if with_netdev:
+        (net / "eth0").mkdir()
+
+
+@pytest.fixture
+def ib_sysfs(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
+    ib = tmp_path / "infiniband"
+    ib.mkdir()
+    monkeypatch.setattr(nccl_module, "_IB_SYSFS", ib)
+    monkeypatch.delenv("NCCL_IB_HCA", raising=False)
+    monkeypatch.delenv("NCCL_IB_DISABLE", raising=False)
+    return ib
+
+
+def test_returns_exact_usable_hcas_when_phantoms_exist(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_1", with_netdev=True)
+    _make_hca(ib_sysfs, "mlx5_10", with_netdev=True)
+    _make_hca(ib_sysfs, "mlx5_2")
+
+    assert get_nccl_ib_env() == {"NCCL_IB_HCA": "=mlx5_1,=mlx5_10"}
+    assert "NCCL_IB_HCA" not in os.environ
+
+
+def test_disables_ib_when_all_hcas_are_phantom(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_0")
+    _make_hca(ib_sysfs, "mlx5_1")
+
+    assert get_nccl_ib_env() == {"NCCL_IB_DISABLE": "1"}
+
+
+def test_returns_no_overrides_without_phantoms(ib_sysfs: Path) -> None:
+    _make_hca(ib_sysfs, "mlx5_0", with_netdev=True)
+
+    assert get_nccl_ib_env() == {}
+
+
+def test_respects_existing_nccl_configuration(ib_sysfs: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+    _make_hca(ib_sysfs, "mlx5_0")
+    monkeypatch.setenv("NCCL_IB_HCA", "mlx5_custom")
+
+    assert get_nccl_ib_env() == {}
diff --git a/plugins/nemo-automodel/src/nemo_automodel_plugin/jobs/jobs.py b/plugins/nemo-automodel/src/nemo_automodel_plugin/jobs/jobs.py
index 0ca2045bb1..85fb6f11a0 100644
--- a/plugins/nemo-automodel/src/nemo_automodel_plugin/jobs/jobs.py
+++ b/plugins/nemo-automodel/src/nemo_automodel_plugin/jobs/jobs.py
@@ -14,11 +14,12 @@
 from typing import ClassVar, cast
 
 from nemo_automodel_plugin.config import get_config
-from nemo_automodel_plugin.schema import AutomodelJobInput, AutomodelJobOutput
+from nemo_automodel_plugin.schema import AutomodelJobInput, AutomodelJobOutput, ValidationError
 from nemo_automodel_plugin.transform import transform_input_to_output
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec
 from nemo_platform_plugin.jobs.docker import validate_gpu_available_for_docker
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
 from nmp.automodel.compile import platform_job_config_compiler
 from nmp.customization_common.contributor.jobs import BaseSubmitJob, require_container_runtime
 from pydantic import BaseModel
@@ -56,7 +57,10 @@ async def compile(
         # Multi-node jobs compile to a gpu_distributed (Volcano) executor, which
         # only exists on Kubernetes; gate here so docker platforms fail fast.
         require_container_runtime(cls.runtime_label, num_nodes=canonical.parallelism.num_nodes)
-        canonical.validate_for_training()
+        try:
+            canonical.validate_for_training()
+        except ValidationError as e:
+            raise PlatformJobCompilationError(str(e)) from e
 
         plugin_config = get_config()
         execution_profile = (
diff --git a/plugins/nemo-automodel/tests/test_jobs.py b/plugins/nemo-automodel/tests/test_jobs.py
new file mode 100644
index 0000000000..4fa89d98c5
--- /dev/null
+++ b/plugins/nemo-automodel/tests/test_jobs.py
@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for AutomodelJob.compile error mapping.
+
+``validate_for_training`` rejects inconsistent parallelism/batch topologies.
+Those are user input errors, so they have to leave ``compile`` as
+``PlatformJobCompilationError`` — the api_factory only maps that type to a
+422, and anything else escapes as a 500.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+from nemo_automodel_plugin.jobs.jobs import AutomodelJob
+from nemo_automodel_plugin.schema import AutomodelJobOutput
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
+
+
+def _make_canonical(**parallelism: Any) -> AutomodelJobOutput:
+    return AutomodelJobOutput.model_validate(
+        {
+            "model": "default/base",
+            "dataset": {"training": "default/train"},
+            "training": {"training_type": "sft"},
+            "schedule": {"epochs": 1},
+            "batch": {"global_batch_size": 4, "micro_batch_size": 1},
+            "optimizer": {},
+            "parallelism": {"num_nodes": 2, "num_gpus_per_node": 8, **parallelism},
+            "output": {"name": "out", "type": "adapter", "fileset": "out-fs"},
+        },
+    )
+
+
+def _compile(canonical: AutomodelJobOutput) -> Any:
+    return asyncio.run(
+        AutomodelJob.compile(
+            workspace="default",
+            spec=canonical,
+            entity_client=object(),
+            job_name=None,
+            async_sdk=object(),
+        ),
+    )
+
+
+class TestCompileValidationErrors:
+    def test_indivisible_batch_raises_compilation_error(self) -> None:
+        # global_batch_size=4 with data_parallel_size=16 (2 nodes x 8 GPUs, TP=1).
+        canonical = _make_canonical(tensor_parallel_size=1)
+        with patch("nemo_automodel_plugin.jobs.jobs.require_container_runtime"):
+            with pytest.raises(PlatformJobCompilationError, match="global_batch_size"):
+                _compile(canonical)
+
+    def test_indivisible_model_parallel_raises_compilation_error(self) -> None:
+        # 16 total GPUs is not divisible by tensor_parallel_size=5.
+        canonical = _make_canonical(tensor_parallel_size=5)
+        with patch("nemo_automodel_plugin.jobs.jobs.require_container_runtime"):
+            with pytest.raises(PlatformJobCompilationError, match="Total GPUs"):
+                _compile(canonical)
diff --git a/plugins/nemo-rl/src/nemo_rl_plugin/jobs/jobs.py b/plugins/nemo-rl/src/nemo_rl_plugin/jobs/jobs.py
index 022c6add5f..5cf21f068f 100644
--- a/plugins/nemo-rl/src/nemo_rl_plugin/jobs/jobs.py
+++ b/plugins/nemo-rl/src/nemo_rl_plugin/jobs/jobs.py
@@ -19,6 +19,7 @@
 
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
 from nemo_rl_plugin.schema import RlJobInput
 from nemo_rl_plugin.transform import transform_input_to_output
 from nmp.customization_common.contributor.jobs import BaseSubmitJob, require_distributed_runtime
@@ -63,7 +64,10 @@ async def compile(
         del entity_client, options
         require_distributed_runtime(cls.runtime_label)
         canonical = spec if isinstance(spec, RlJobOutput) else RlJobOutput.model_validate(spec.model_dump())
-        canonical.validate_for_training()
+        try:
+            canonical.validate_for_training()
+        except ValueError as e:
+            raise PlatformJobCompilationError(str(e)) from e
 
         # Leave ``None`` when unset so the compiler can default per topology.
         execution_profile = canonical.training.execution_profile or profile
diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml
index 829a0150ee..f07d6e4b4d 100644
--- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml
+++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml
@@ -10172,7 +10172,7 @@ components:
           type: string
           title: Volume Permissions Image
           description: Docker image used to set permissions on the volume
-          default: busybox
+          default: docker.io/library/busybox:stable
         additional_volume_mounts:
           items:
             $ref: '#/components/schemas/DockerVolumeMount'
@@ -13177,7 +13177,7 @@ components:
           type: string
           title: Volume Permissions Image
           description: Image used to set volume permissions
-          default: busybox
+          default: docker.io/library/busybox:stable
         additional_volumes:
           items:
             $ref: '#/components/schemas/KubernetesVolume'
diff --git a/services/automodel/src/nmp/automodel/tasks/training/backends/backend.py b/services/automodel/src/nmp/automodel/tasks/training/backends/backend.py
index 901b8a644d..f1dcc08776 100644
--- a/services/automodel/src/nmp/automodel/tasks/training/backends/backend.py
+++ b/services/automodel/src/nmp/automodel/tasks/training/backends/backend.py
@@ -2,6 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import logging
+import os
 import signal
 import subprocess
 import threading
@@ -24,6 +25,7 @@
 )
 from nmp.automodel.tasks.training.utils import generate_torchrun_flags_from_env
 from nmp.customization_common.service.context import NMPJobContext
+from nmp.customization_common.training.nccl import get_nccl_ib_env
 
 from .checkpoints import ModelType, find_best_checkpoint, process_checkpoint
 from .config import compile_automodel_config
@@ -104,12 +106,16 @@ def cleanup(signum, frame):
 
         start_time = time.time()
 
+        training_env = os.environ.copy()
+        if customizer_config.parallelism.num_nodes > 1:
+            training_env.update(get_nccl_ib_env())
         training_process = subprocess.Popen(
             command,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             text=True,
             bufsize=1,  # Line buffered
+            env=training_env,
         )
 
         # Start reader thread to capture output without blocking
diff --git a/services/core/jobs/src/nmp/core/jobs/app/constants.py b/services/core/jobs/src/nmp/core/jobs/app/constants.py
index e5a738c9d3..03166c08c9 100644
--- a/services/core/jobs/src/nmp/core/jobs/app/constants.py
+++ b/services/core/jobs/src/nmp/core/jobs/app/constants.py
@@ -22,7 +22,7 @@
 JOB_EXECUTION_PROFILE_LABEL = "nmp.nvidia.com/job_execution_profile"
 
 NEMO_JOB_TASK_CONTAINER_NAME = "nemo-job-task"
-DEFAULT_VOLUME_PERMISSIONS_IMAGE = "busybox"
+DEFAULT_VOLUME_PERMISSIONS_IMAGE = "docker.io/library/busybox:stable"
 
 KUBE_JOB_SELECTOR_LABELS = {
     "app": "nemo-job",
diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py
index a74f80ba57..bed8f82ae6 100644
--- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py
+++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py
@@ -65,6 +65,7 @@
 from nmp.common.observability import start_span_with_ctx
 from nmp.common.resources import SharedResourceManager
 from nmp.core.jobs.app.constants import (
+    DEFAULT_VOLUME_PERMISSIONS_IMAGE,
     JOB_ATTEMPT_ID_LABEL,
     JOB_CONTROLLER_INSTANCE_ID_LABEL,
     JOB_EXECUTION_BACKEND_LABEL,
@@ -139,8 +140,6 @@ def k8s_shm_quantity_to_docker(quantity: str) -> str:
     return q
 
 
-DEFAULT_VOLUME_PERMISSIONS_IMAGE = "busybox"
-
 NEMO_JOBS_IMAGE_REGISTRY_PASSWORD = os.getenv("NEMO_JOBS_IMAGE_REGISTRY_PASSWORD")
 NEMO_JOBS_IMAGE_REGISTRY = os.getenv("NEMO_JOBS_IMAGE_REGISTRY")
 NEMO_JOBS_IMAGE_REGISTRY_USER_NAME = os.getenv("NEMO_JOBS_IMAGE_REGISTRY_USER_NAME")
diff --git a/services/rl/src/nmp/rl/tasks/training/backends/nemo_rl/backend.py b/services/rl/src/nmp/rl/tasks/training/backends/nemo_rl/backend.py
index fbb75a5b07..6e59ecd98f 100644
--- a/services/rl/src/nmp/rl/tasks/training/backends/nemo_rl/backend.py
+++ b/services/rl/src/nmp/rl/tasks/training/backends/nemo_rl/backend.py
@@ -18,6 +18,7 @@
 
 from nemo_rl.utils.checkpoint import CheckpointingConfig, CheckpointManager
 from nmp.customization_common.service.context import NMPJobContext
+from nmp.customization_common.training.nccl import get_nccl_ib_env
 from nmp.customization_common.training.progress import JobsServiceProgressReporter
 from nmp.rl.app.jobs.training.schemas import (
     CheckpointFormat,
@@ -130,6 +131,8 @@ def execute_training(
             "BASE_LOG_DIR": str(workspace_dir),
             "GPUS_PER_NODE": str(customizer_config.parallelism.num_gpus_per_node),
         }
+        if customizer_config.parallelism.num_nodes > 1:
+            env_overrides.update(get_nccl_ib_env())
         # MLflow integration (if configured)
         if customizer_config.integrations and customizer_config.integrations.mlflow:
             mlflow_config = customizer_config.integrations.mlflow