diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx index b4c32ba8d5..9c2ea8f5cb 100644 --- a/docs/set-up/config-reference.mdx +++ b/docs/set-up/config-reference.mdx @@ -224,6 +224,8 @@ jobs: cleanup_completed_jobs_immediately: true # Path to the jobs launcher tool | default: '/tools/jobs-launcher' launcher_tool_path: /tools/jobs-launcher + # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + default_task_image: # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables. env: {} # Docker storage configuration @@ -250,6 +252,8 @@ jobs: cleanup_completed_jobs_immediately: true # Path to the jobs launcher tool | default: '/tools/jobs-launcher' launcher_tool_path: /tools/jobs-launcher + # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + default_task_image: # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables. env: {} # Kubernetes namespace to submit the job to. If not set, it will be determined from the environment. @@ -322,6 +326,8 @@ jobs: cleanup_completed_jobs_immediately: true # Path to the jobs launcher tool | default: '/tools/jobs-launcher' launcher_tool_path: /tools/jobs-launcher + # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + default_task_image: # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables. env: {} # Kubernetes namespace to submit the job to. If not set, it will be determined from the environment. @@ -402,6 +408,8 @@ jobs: cleanup_completed_jobs_immediately: false # Path to the jobs launcher tool | default: '/tools/jobs-launcher' launcher_tool_path: /tools/jobs-launcher + # Default container image for job task pods. Used when a job step omits container.image. When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + default_task_image: # Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables. env: {} # Root directory for subprocess job state, config, storage, and logs. | default: '/tmp/nmp-subprocess-jobs' diff --git a/e2e/test_jobs.py b/e2e/test_jobs.py index 12255b52df..2bffe386c4 100644 --- a/e2e/test_jobs.py +++ b/e2e/test_jobs.py @@ -1,9 +1,9 @@ -"""E2E tests for platform jobs via the subprocess executor. +"""E2E tests for platform jobs. -These tests submit jobs with CPUExecutionProviderSpec (container image + command). -In subprocess mode, the jobs service translates cpu/default steps to subprocess -steps automatically — the container image is discarded and the command runs -directly on the host. +These tests submit jobs with CPUExecutionProviderSpec (container + command). +The container image is omitted so that: +- On subprocess mode, the cpu→subprocess translation discards it anyway. +- On Kubernetes/Docker, the execution profile's default_task_image is used. Ported from Platform-Deploy e2e/test_jobs.py, adapted for the SDK's TypedDict param types and filtered to tests that work without Docker. @@ -17,10 +17,6 @@ JOB_SOURCE = "e2e-test-jobs" -# The image is discarded by the cpu→subprocess translation, but must be -# syntactically valid for the API to accept the CPUExecutionProvider. -PLACEHOLDER_IMAGE = "placeholder:unused" - pytestmark = [pytest.mark.timeout(600)] @@ -62,7 +58,6 @@ def test_basic_platform_job_lifecycle(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["echo", "Hello from e2e test!"], }, }, @@ -106,7 +101,6 @@ def test_job_logs_across_multiple_batches(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["sh", "-c", log_command], }, }, @@ -146,7 +140,6 @@ def test_job_config_is_readable(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["sh", "-c", "echo 'Step config:'; cat $NEMO_JOB_STEP_CONFIG_FILE_PATH;"], }, }, @@ -181,7 +174,6 @@ def test_job_passing_data_between_steps(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": [ "sh", "-c", @@ -195,7 +187,6 @@ def test_job_passing_data_between_steps(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": [ "sh", "-c", @@ -237,7 +228,6 @@ def test_job_using_secret_environment_variable(sdk: NeMoPlatform, workspace: str "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["sh", "-c", 'echo "Secret value is: $SECRET_ENV_VAR"'], }, }, @@ -275,7 +265,6 @@ def test_job_with_expected_failure(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["sh", "-c", "echo 'This step will fail'; exit 1;"], }, }, @@ -305,7 +294,6 @@ def test_job_cancel_immediately(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["sh", "-c", "sleep 60"], }, }, @@ -335,7 +323,6 @@ def test_job_cancel_once_active(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["sh", "-c", "sleep 300"], }, }, @@ -376,7 +363,6 @@ def test_job_pause_resume(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["sh", "-c", "sleep 300"], }, }, @@ -418,7 +404,6 @@ def test_job_pause_and_cancel(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": ["sh", "-c", "sleep 300"], }, }, @@ -455,7 +440,6 @@ def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": [ "sh", "-c", @@ -470,7 +454,6 @@ def test_job_using_additional_volume(sdk: NeMoPlatform, workspace: str): "executor": { "provider": "cpu", "container": { - "image": PLACEHOLDER_IMAGE, "command": [ "sh", "-c", diff --git a/openapi/ga/individual/platform.openapi.yaml b/openapi/ga/individual/platform.openapi.yaml index 386598e5cf..5b4f4b0bcd 100644 --- a/openapi/ga/individual/platform.openapi.yaml +++ b/openapi/ga/individual/platform.openapi.yaml @@ -8597,8 +8597,9 @@ components: ContainerSpec: properties: image: - type: string title: Image + type: string + minLength: 1 entrypoint: items: type: string @@ -8610,8 +8611,6 @@ components: type: array title: Command type: object - required: - - image title: ContainerSpec description: 'Specification for a container configuration. @@ -9355,6 +9354,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -11771,6 +11777,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -11888,6 +11901,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -16537,6 +16557,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -17553,6 +17580,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string diff --git a/openapi/ga/openapi.yaml b/openapi/ga/openapi.yaml index 386598e5cf..5b4f4b0bcd 100644 --- a/openapi/ga/openapi.yaml +++ b/openapi/ga/openapi.yaml @@ -8597,8 +8597,9 @@ components: ContainerSpec: properties: image: - type: string title: Image + type: string + minLength: 1 entrypoint: items: type: string @@ -8610,8 +8611,6 @@ components: type: array title: Command type: object - required: - - image title: ContainerSpec description: 'Specification for a container configuration. @@ -9355,6 +9354,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -11771,6 +11777,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -11888,6 +11901,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -16537,6 +16557,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -17553,6 +17580,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string diff --git a/openapi/openapi.yaml b/openapi/openapi.yaml index 386598e5cf..5b4f4b0bcd 100644 --- a/openapi/openapi.yaml +++ b/openapi/openapi.yaml @@ -8597,8 +8597,9 @@ components: ContainerSpec: properties: image: - type: string title: Image + type: string + minLength: 1 entrypoint: items: type: string @@ -8610,8 +8611,6 @@ components: type: array title: Command type: object - required: - - image title: ContainerSpec description: 'Specification for a container configuration. @@ -9355,6 +9354,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -11771,6 +11777,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -11888,6 +11901,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -16537,6 +16557,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -17553,6 +17580,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string diff --git a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml index 386598e5cf..5b4f4b0bcd 100644 --- a/sdk/python/nemo-platform/.nmpcontext/openapi.yaml +++ b/sdk/python/nemo-platform/.nmpcontext/openapi.yaml @@ -8597,8 +8597,9 @@ components: ContainerSpec: properties: image: - type: string title: Image + type: string + minLength: 1 entrypoint: items: type: string @@ -8610,8 +8611,6 @@ components: type: array title: Command type: object - required: - - image title: ContainerSpec description: 'Specification for a container configuration. @@ -9355,6 +9354,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -11771,6 +11777,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -11888,6 +11901,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -16537,6 +16557,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string @@ -17553,6 +17580,13 @@ components: title: Launcher Tool Path description: Path to the jobs launcher tool default: /tools/jobs-launcher + default_task_image: + title: Default Task Image + description: Default container image for job task pods. Used when a job + step omits container.image. When unset, falls back to the platform CPU + tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + type: string + minLength: 1 env: additionalProperties: type: string diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/container_spec.py b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/container_spec.py index 266b0f06ed..6d2de8677e 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/container_spec.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/container_spec.py @@ -28,8 +28,8 @@ class ContainerSpec(BaseModel): Defines the container image and related configuration for job execution. """ - image: str - command: Optional[List[str]] = None entrypoint: Optional[List[str]] = None + + image: Optional[str] = None diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/container_spec_param.py b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/container_spec_param.py index 5497499027..23ac07cc30 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/container_spec_param.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/container_spec_param.py @@ -17,7 +17,7 @@ from __future__ import annotations -from typing_extensions import Required, TypedDict +from typing_extensions import TypedDict from ..._types import SequenceNotStr @@ -30,8 +30,8 @@ class ContainerSpecParam(TypedDict, total=False): Defines the container image and related configuration for job execution. """ - image: Required[str] - command: SequenceNotStr[str] entrypoint: SequenceNotStr[str] + + image: str diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/docker_job_execution_profile_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/docker_job_execution_profile_config.py index 1a6d2602ad..cdce4354fa 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/docker_job_execution_profile_config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/docker_job_execution_profile_config.py @@ -29,6 +29,14 @@ class DockerJobExecutionProfileConfig(BaseModel): cleanup_completed_jobs_immediately: Optional[bool] = None + default_task_image: Optional[str] = None + """Default container image for job task pods. + + Used when a job step omits container.image. When unset, falls back to the + platform CPU tasks image + (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + """ + env: Optional[Dict[str, str]] = None """Optional env vars applied to all jobs (e.g. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/job_execution_profile_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/job_execution_profile_config.py index f368ee1589..d30d547c45 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/job_execution_profile_config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/job_execution_profile_config.py @@ -25,6 +25,14 @@ class JobExecutionProfileConfig(BaseModel): cleanup_completed_jobs_immediately: Optional[bool] = None + default_task_image: Optional[str] = None + """Default container image for job task pods. + + Used when a job step omits container.image. When unset, falls back to the + platform CPU tasks image + (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + """ + env: Optional[Dict[str, str]] = None """Optional env vars applied to all jobs (e.g. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/kubernetes_job_execution_profile_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/kubernetes_job_execution_profile_config.py index 34ff6d5ca9..81f5941f63 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/kubernetes_job_execution_profile_config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/kubernetes_job_execution_profile_config.py @@ -34,6 +34,14 @@ class KubernetesJobExecutionProfileConfig(BaseModel): cleanup_completed_jobs_immediately: Optional[bool] = None + default_task_image: Optional[str] = None + """Default container image for job task pods. + + Used when a job step omits container.image. When unset, falls back to the + platform CPU tasks image + (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + """ + env: Optional[Dict[str, str]] = None """Optional env vars applied to all jobs (e.g. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/subprocess_job_execution_profile_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/subprocess_job_execution_profile_config.py index b48dd4350d..8969bb951f 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/subprocess_job_execution_profile_config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/subprocess_job_execution_profile_config.py @@ -26,6 +26,14 @@ class SubprocessJobExecutionProfileConfig(BaseModel): cleanup_completed_jobs_immediately: Optional[bool] = None """Keep subprocess working directories by default so runs remain inspectable.""" + default_task_image: Optional[str] = None + """Default container image for job task pods. + + Used when a job step omits container.image. When unset, falls back to the + platform CPU tasks image + (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + """ + env: Optional[Dict[str, str]] = None """Optional env vars applied to all jobs (e.g. diff --git a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/volcano_job_execution_profile_config.py b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/volcano_job_execution_profile_config.py index 0c04f84278..c4ace352c0 100644 --- a/sdk/python/nemo-platform/src/nemo_platform/types/jobs/volcano_job_execution_profile_config.py +++ b/sdk/python/nemo-platform/src/nemo_platform/types/jobs/volcano_job_execution_profile_config.py @@ -34,6 +34,14 @@ class VolcanoJobExecutionProfileConfig(BaseModel): cleanup_completed_jobs_immediately: Optional[bool] = None + default_task_image: Optional[str] = None + """Default container image for job task pods. + + Used when a job step omits container.image. When unset, falls back to the + platform CPU tasks image + (platform.image_registry/nmp-cpu-tasks:platform.image_tag). + """ + enable_multi_node_networking: Optional[bool] = None """Enable multi-node networking injection. diff --git a/sdk/python/nemo-platform/tests/api_resources/test_jobs.py b/sdk/python/nemo-platform/tests/api_resources/test_jobs.py index dce491ed23..16c770f140 100644 --- a/sdk/python/nemo-platform/tests/api_resources/test_jobs.py +++ b/sdk/python/nemo-platform/tests/api_resources/test_jobs.py @@ -52,7 +52,7 @@ def test_method_create(self, client: NeMoPlatform) -> None: "steps": [ { "executor": { - "container": {"image": "image"}, + "container": {}, "provider": "cpu", }, "name": "preprocess", @@ -74,9 +74,9 @@ def test_method_create_with_all_params(self, client: NeMoPlatform) -> None: { "executor": { "container": { - "image": "image", "command": ["string"], "entrypoint": ["string"], + "image": "x", }, "profile": "profile", "provider": "cpu", @@ -126,7 +126,7 @@ def test_raw_response_create(self, client: NeMoPlatform) -> None: "steps": [ { "executor": { - "container": {"image": "image"}, + "container": {}, "provider": "cpu", }, "name": "preprocess", @@ -151,7 +151,7 @@ def test_streaming_response_create(self, client: NeMoPlatform) -> None: "steps": [ { "executor": { - "container": {"image": "image"}, + "container": {}, "provider": "cpu", }, "name": "preprocess", @@ -179,7 +179,7 @@ def test_path_params_create(self, client: NeMoPlatform) -> None: "steps": [ { "executor": { - "container": {"image": "image"}, + "container": {}, "provider": "cpu", }, "name": "preprocess", @@ -746,7 +746,7 @@ async def test_method_create(self, async_client: AsyncNeMoPlatform) -> None: "steps": [ { "executor": { - "container": {"image": "image"}, + "container": {}, "provider": "cpu", }, "name": "preprocess", @@ -768,9 +768,9 @@ async def test_method_create_with_all_params(self, async_client: AsyncNeMoPlatfo { "executor": { "container": { - "image": "image", "command": ["string"], "entrypoint": ["string"], + "image": "x", }, "profile": "profile", "provider": "cpu", @@ -820,7 +820,7 @@ async def test_raw_response_create(self, async_client: AsyncNeMoPlatform) -> Non "steps": [ { "executor": { - "container": {"image": "image"}, + "container": {}, "provider": "cpu", }, "name": "preprocess", @@ -845,7 +845,7 @@ async def test_streaming_response_create(self, async_client: AsyncNeMoPlatform) "steps": [ { "executor": { - "container": {"image": "image"}, + "container": {}, "provider": "cpu", }, "name": "preprocess", @@ -873,7 +873,7 @@ async def test_path_params_create(self, async_client: AsyncNeMoPlatform) -> None "steps": [ { "executor": { - "container": {"image": "image"}, + "container": {}, "provider": "cpu", }, "name": "preprocess", diff --git a/services/core/jobs/src/nmp/core/jobs/app/providers.py b/services/core/jobs/src/nmp/core/jobs/app/providers.py index 6d5c5486a9..c57f6c08bd 100644 --- a/services/core/jobs/src/nmp/core/jobs/app/providers.py +++ b/services/core/jobs/src/nmp/core/jobs/app/providers.py @@ -18,8 +18,8 @@ class ContainerSpec(BaseModel): Defines the container image and related configuration for job execution. """ - image: str - """The container image to use for execution""" + image: str | None = Field(default=None, min_length=1) + """The container image to use for execution. When omitted, resolved from the execution profile's default_task_image or the platform CPU tasks image.""" entrypoint: list[str] = Field(default_factory=list) """The entrypoint for the container as a list of strings (e.g., ['python', 'script.py']). This overrides a container's default entrypoint (e.g. ENTRYPOINT in Docker) if provided.""" diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/base.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/base.py index 54404b11d7..0bf9088313 100644 --- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/base.py +++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/base.py @@ -91,6 +91,12 @@ class JobExecutionProfileConfig(BaseModel): ttl_seconds_after_finished: int = 60 * 60 # 1 hour cleanup_completed_jobs_immediately: bool = True launcher_tool_path: str = Field(default="/tools/jobs-launcher", description="Path to the jobs launcher tool") + default_task_image: str | None = Field( + default=None, + min_length=1, + description="Default container image for job task pods. Used when a job step omits container.image. " + "When unset, falls back to the platform CPU tasks image (platform.image_registry/nmp-cpu-tasks:platform.image_tag).", + ) env: dict[str, str] = Field( default_factory=dict, description="Optional env vars applied to all jobs (e.g. HOME=/tmp). Keys must not conflict with platform-reserved names. Job steps may override these variables.", @@ -106,6 +112,22 @@ def validate_env_no_reserved_names(self) -> JobExecutionProfileConfig: return self +_DEFAULT_TASK_IMAGE_NAME = "nmp-cpu-tasks" + + +def resolve_task_image(container_image: str | None, default_task_image: str | None) -> str: + """Resolve the container image for a job task. + + Priority: + 1. Explicit container.image from the job step + 2. default_task_image from the execution profile config + 3. Platform CPU tasks image derived from platform.image_registry / image_tag + """ + from nemo_platform_plugin.jobs.image import get_qualified_image + + return container_image or default_task_image or get_qualified_image(_DEFAULT_TASK_IMAGE_NAME) + + def resolve_gpu_job_shm_size( executor_resources: ComputeResources | None, profile_resources: ComputeResources | None, diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py index 74ce48e097..7353358f08 100644 --- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py +++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/docker.py @@ -72,6 +72,7 @@ JobUpdate, get_logs_endpoint_from_fileset, resolve_gpu_job_shm_size, + resolve_task_image, staleness_error_message, ) from nmp.core.jobs.controllers.backends.exceptions import ( @@ -680,11 +681,14 @@ def schedule_single_container( else: labels[JOB_USES_PERSISTENT_STORAGE_LABEL] = "false" + task_image = resolve_task_image( + executor_config.container.image, self._execution_profile_config.default_task_image + ) container_args = { "name": self.name_for_step(step), "entrypoint": executor_config.container.entrypoint or [], "command": executor_config.container.command or [], - "image": executor_config.container.image, + "image": task_image, "labels": labels, "log_config": log_config, "environment": env, diff --git a/services/core/jobs/src/nmp/core/jobs/controllers/backends/kubernetes/common.py b/services/core/jobs/src/nmp/core/jobs/controllers/backends/kubernetes/common.py index d1465d1f56..30f24e4991 100644 --- a/services/core/jobs/src/nmp/core/jobs/controllers/backends/kubernetes/common.py +++ b/services/core/jobs/src/nmp/core/jobs/controllers/backends/kubernetes/common.py @@ -56,6 +56,7 @@ JobExecutionProfileConfig, get_logs_endpoint_from_fileset, resolve_gpu_job_shm_size, + resolve_task_image, ) from nmp.core.jobs.controllers.backends.exceptions import FailedToScheduleError, JobStorageError from pydantic import BaseModel, Field, model_validator @@ -1102,10 +1103,14 @@ def create_pod_template_spec( for cmd in container.entrypoint or []: command.append(cmd) + # Resolve the task image: explicit container.image takes precedence, + # then the profile's default_task_image, then platform CPU tasks image fallback. + task_image = resolve_task_image(container.image, config.default_task_image) + # Main job container job_container = client.V1Container( name=NEMO_JOB_TASK_CONTAINER_NAME, - image=container.image, + image=task_image, command=command, args=container.command, env=env, diff --git a/services/core/jobs/tests/controllers/test_base.py b/services/core/jobs/tests/controllers/test_base.py index 759e1e5f81..6d85637939 100644 --- a/services/core/jobs/tests/controllers/test_base.py +++ b/services/core/jobs/tests/controllers/test_base.py @@ -11,7 +11,7 @@ from nmp.core.jobs.api.v2.jobs.schemas import PlatformJobStepWithContext from nmp.core.jobs.app.providers import ContainerSpec, CPUExecutionProvider from nmp.core.jobs.app.schemas import PlatformJobStepSpec, StepLifecycle -from nmp.core.jobs.controllers.backends.base import get_logs_endpoint_from_fileset +from nmp.core.jobs.controllers.backends.base import get_logs_endpoint_from_fileset, resolve_task_image from nmp.core.jobs.controllers.backends.test import MockKubernetesCPUJobBackend @@ -316,3 +316,21 @@ def test_returns_false_when_task_missing_updated_at(self): ) assert backend.check_step_is_stale(step) is False + + +class TestResolveTaskImage: + """Tests for resolve_task_image.""" + + def test_explicit_image_takes_precedence(self): + assert resolve_task_image("my-image:v1", "default-image:latest") == "my-image:v1" + + def test_falls_back_to_default_task_image(self): + assert resolve_task_image(None, "default-image:latest") == "default-image:latest" + + def test_explicit_image_without_default(self): + assert resolve_task_image("my-image:v1", None) == "my-image:v1" + + def test_falls_back_to_platform_cpu_tasks_image_when_both_none(self): + with patch("nemo_platform_plugin.jobs.image.get_platform_config") as mock_config: + mock_config.return_value = MagicMock(image_registry="my-registry", image_tag="v1.0") + assert resolve_task_image(None, None) == "my-registry/nmp-cpu-tasks:v1.0"