soluwalana · Jul 27, 2026 · Jul 27, 2026 · Jul 27, 2026 · Jul 27, 2026 · Jul 27, 2026
@@ -163,7 +163,7 @@ runs:
         HELM_VALUES: ${{ inputs['helm-values'] }}
         REQUIRE_NMP_E2E_IMAGES: "true"
         POSTGRES_IMAGE: docker.io/library/postgres
-        BUSYBOX_IMAGE: docker.io/library/busybox
+        BUSYBOX_IMAGE: docker.io/library/busybox:stable
         GITHUB_TOKEN: ${{ inputs['kind-image-pull-token'] }}
       run: |
         if ! e2e/k8s/scripts/install_helm_e2e.sh; then

@@ -268,8 +268,8 @@ jobs:
       storage:
         # Name of the Docker volume for persistent storage | default: 'nemo-jobs-storage'
         volume_name: nemo-jobs-storage
-        # Docker image used to set permissions on the volume | default: 'busybox'
-        volume_permissions_image: busybox
+        # Docker image used to set permissions on the volume | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # List of additional Docker volume mounts for the job
         additional_volume_mounts: []
       # Docker networking configuration
@@ -358,8 +358,8 @@ jobs:
       storage:
         # Persistent Volume Claim Name to use for job storage. | default: ''
         pvc_name: ''
-        # Image used to set volume permissions | default: 'busybox'
-        volume_permissions_image: busybox
+        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # Additional volumes to mount
         additional_volumes: []
         # Additional volume mounts
@@ -436,8 +436,8 @@ jobs:
       storage:
         # Persistent Volume Claim Name to use for job storage. | default: ''
         pvc_name: ''
-        # Image used to set volume permissions | default: 'busybox'
-        volume_permissions_image: busybox
+        # Image used to set volume permissions | default: 'docker.io/library/busybox:stable'
+        volume_permissions_image: docker.io/library/busybox:stable
         # Additional volumes to mount
         additional_volumes: []
         # Additional volume mounts

@@ -35,4 +35,4 @@ platformConfig:
 core:
   storage:
     storageClass: oci-nfs
-    volumePermissionsImage: docker.io/library/busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
@@ -11,7 +11,7 @@ core:
   storage:
     accessModes:
       - ReadWriteOnce
-    volumePermissionsImage: docker.io/library/busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: false

@@ -15,7 +15,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-    volumePermissionsImage: busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 envoyProxy:
   enabled: false

@@ -17,7 +17,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-    volumePermissionsImage: busybox
+    volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: true

@@ -12,7 +12,7 @@ postgresql:
 core:
   storage:
     storageClass: standard
-  volumePermissionsImage: docker.io/library/busybox
+  volumePermissionsImage: docker.io/library/busybox:stable
 
 ingress:
   enabled: true

@@ -154,7 +154,7 @@ secrets will not decrypt with a new key.
 | core.storage.existingPersistentVolumeName | string | `""` | If set, pods will mount this persistent volume for job-scoped storage and we will not create a new persistent volume claim. |
 | core.storage.size | string | `"200Gi"` | size of the persistent volume claim used for persistent storage |
 | core.storage.storageClass | string | `""` | Which storageClass to use when creating a new persistent volume claim. Empty string uses the cluster's default StorageClass. |
-| core.storage.volumePermissionsImage | string | `"docker.io/library/busybox"` | volumePermissionsImage is the image used to set permissions on the volume |
+| core.storage.volumePermissionsImage | string | `"docker.io/library/busybox:stable"` | volumePermissionsImage is the image used to set permissions on the volume |
 | core.telemetry | object | `{}` | OpenTelemetry configuration overrides for the platform deployment. |
 | env | object | `{}` | Environment variables that will be applied to every deployment pod. Uses a simple key value map structure like MY_ENV_VAR: the-key and works with valueFrom as well. |
 | envFromSecret | string | `""` | Optional. Name of an existing Kubernetes Secret to load as env vars (envFrom) for the API pod. When set, the chart does not create or generate the default api-env Secret; use your own Secret (for example, from Vault or sealed-secrets). |

@@ -720,7 +720,7 @@ core:
     # -- size of the persistent volume claim used for persistent storage
     size: 200Gi
     # -- volumePermissionsImage is the image used to set permissions on the volume
-    volumePermissionsImage: "docker.io/library/busybox"
+    volumePermissionsImage: "docker.io/library/busybox:stable"
     # -- Annotations to add to the persistent volume claim
     annotations: {}
 

@@ -39,7 +39,7 @@
 from pydantic import BaseModel, ConfigDict, Field, model_validator
 
 # Default image used to set filesystem permissions on job storage volumes.
-DEFAULT_VOLUME_PERMISSIONS_IMAGE = "busybox"
+DEFAULT_VOLUME_PERMISSIONS_IMAGE = "docker.io/library/busybox:stable"
 JOB_LOGS_ENDPOINT_ENVVAR = "NMP_JOB_LOGS_ENDPOINT"
 
 # Env var names set by the platform during job creation; user-provided profile

@@ -14,11 +14,12 @@
 from typing import ClassVar, cast
 
 from nemo_automodel_plugin.config import get_config
-from nemo_automodel_plugin.schema import AutomodelJobInput, AutomodelJobOutput
+from nemo_automodel_plugin.schema import AutomodelJobInput, AutomodelJobOutput, ValidationError
 from nemo_automodel_plugin.transform import transform_input_to_output
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec
 from nemo_platform_plugin.jobs.docker import validate_gpu_available_for_docker
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
 from nmp.automodel.compile import platform_job_config_compiler
 from nmp.customization_common.contributor.jobs import BaseSubmitJob, require_container_runtime
 from pydantic import BaseModel
@@ -56,7 +57,10 @@ async def compile(
         # Multi-node jobs compile to a gpu_distributed (Volcano) executor, which
         # only exists on Kubernetes; gate here so docker platforms fail fast.
         require_container_runtime(cls.runtime_label, num_nodes=canonical.parallelism.num_nodes)
-        canonical.validate_for_training()
+        try:
+            canonical.validate_for_training()
+        except ValidationError as e:
+            raise PlatformJobCompilationError(str(e)) from e
 
         plugin_config = get_config()
         execution_profile = (

@@ -0,0 +1,64 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tests for AutomodelJob.compile error mapping.
+
+``validate_for_training`` rejects inconsistent parallelism/batch topologies.
+Those are user input errors, so they have to leave ``compile`` as
+``PlatformJobCompilationError`` — the api_factory only maps that type to a
+422, and anything else escapes as a 500.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+from nemo_automodel_plugin.jobs.jobs import AutomodelJob
+from nemo_automodel_plugin.schema import AutomodelJobOutput
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
+
+
+def _make_canonical(**parallelism: Any) -> AutomodelJobOutput:
+    return AutomodelJobOutput.model_validate(
+        {
+            "model": "default/base",
+            "dataset": {"training": "default/train"},
+            "training": {"training_type": "sft"},
+            "schedule": {"epochs": 1},
+            "batch": {"global_batch_size": 4, "micro_batch_size": 1},
+            "optimizer": {},
+            "parallelism": {"num_nodes": 2, "num_gpus_per_node": 8, **parallelism},
+            "output": {"name": "out", "type": "adapter", "fileset": "out-fs"},
+        },
+    )
+
+
+def _compile(canonical: AutomodelJobOutput) -> Any:
+    return asyncio.run(
+        AutomodelJob.compile(
+            workspace="default",
+            spec=canonical,
+            entity_client=object(),
+            job_name=None,
+            async_sdk=object(),
+        ),
+    )
+
+
+class TestCompileValidationErrors:
+    def test_indivisible_batch_raises_compilation_error(self) -> None:
+        # global_batch_size=4 with data_parallel_size=16 (2 nodes x 8 GPUs, TP=1).
+        canonical = _make_canonical(tensor_parallel_size=1)
+        with patch("nemo_automodel_plugin.jobs.jobs.require_container_runtime"):
+            with pytest.raises(PlatformJobCompilationError, match="global_batch_size"):
+                _compile(canonical)
+
+    def test_indivisible_model_parallel_raises_compilation_error(self) -> None:
+        # 16 total GPUs is not divisible by tensor_parallel_size=5.
+        canonical = _make_canonical(tensor_parallel_size=5)
+        with patch("nemo_automodel_plugin.jobs.jobs.require_container_runtime"):
+            with pytest.raises(PlatformJobCompilationError, match="Total GPUs"):
+                _compile(canonical)
@@ -19,6 +19,7 @@
 
 from nemo_platform import AsyncNeMoPlatform
 from nemo_platform_plugin.jobs.api_factory import PlatformJobSpec
+from nemo_platform_plugin.jobs.exceptions import PlatformJobCompilationError
 from nemo_rl_plugin.schema import RlJobInput
 from nemo_rl_plugin.transform import transform_input_to_output
 from nmp.customization_common.contributor.jobs import BaseSubmitJob, require_distributed_runtime
@@ -63,7 +64,10 @@ async def compile(
         del entity_client, options
         require_distributed_runtime(cls.runtime_label)
         canonical = spec if isinstance(spec, RlJobOutput) else RlJobOutput.model_validate(spec.model_dump())
-        canonical.validate_for_training()
+        try:
+            canonical.validate_for_training()
+        except ValueError as e:
+            raise PlatformJobCompilationError(str(e)) from e
 
         # Leave ``None`` when unset so the compiler can default per topology.
         execution_profile = canonical.training.execution_profile or profile

@@ -22,7 +22,7 @@
 JOB_EXECUTION_PROFILE_LABEL = "nmp.nvidia.com/job_execution_profile"
 
 NEMO_JOB_TASK_CONTAINER_NAME = "nemo-job-task"
-DEFAULT_VOLUME_PERMISSIONS_IMAGE = "busybox"
+DEFAULT_VOLUME_PERMISSIONS_IMAGE = "docker.io/library/busybox:stable"
 
 KUBE_JOB_SELECTOR_LABELS = {
     "app": "nemo-job",

@@ -65,6 +65,7 @@
 from nmp.common.observability import start_span_with_ctx
 from nmp.common.resources import SharedResourceManager
 from nmp.core.jobs.app.constants import (
+    DEFAULT_VOLUME_PERMISSIONS_IMAGE,
     JOB_ATTEMPT_ID_LABEL,
     JOB_CONTROLLER_INSTANCE_ID_LABEL,
     JOB_EXECUTION_BACKEND_LABEL,
@@ -139,8 +140,6 @@ def k8s_shm_quantity_to_docker(quantity: str) -> str:
     return q
 
 
-DEFAULT_VOLUME_PERMISSIONS_IMAGE = "busybox"
-
 NEMO_JOBS_IMAGE_REGISTRY_PASSWORD = os.getenv("NEMO_JOBS_IMAGE_REGISTRY_PASSWORD")
 NEMO_JOBS_IMAGE_REGISTRY = os.getenv("NEMO_JOBS_IMAGE_REGISTRY")
 NEMO_JOBS_IMAGE_REGISTRY_USER_NAME = os.getenv("NEMO_JOBS_IMAGE_REGISTRY_USER_NAME")