From b840dd51816567be9dcaa2997b8f27c0a625a2db Mon Sep 17 00:00:00 2001 From: Ben McCown Date: Fri, 17 Jul 2026 11:43:06 -0600 Subject: [PATCH 1/3] test(e2e): add nemo-deployments plugin e2e tests for docker and k8s backends Exercise the nemo-deployments plugin's own public API (DeploymentConfig / Deployment / Volume CRUD) end to end through the reconcile controller on both the Docker and Kubernetes executor backends, mirroring the existing nemo-agents deployment e2e structure. - deployments_helpers.py: backend-agnostic scenario cores driven via sdk._client against /apis/deployments/v2/... (service->READY, job->SUCCEEDED, and a volume provision+mount+read-back round-trip). Workload image refs default to docker.io/library/alpine|nginx and are env-overridable, matching the POSTGRES_IMAGE/BUSYBOX_IMAGE knobs the kind e2e install already exposes. - test_nemo_deployments_docker.py: subprocess harness + docker executor (subprocess_only), skips cleanly without a reachable Docker daemon. - test_nemo_deployments_k8s.py: external kind cluster + k8s executor (container_only), wider timeouts for pod scheduling / PVC binding. - configs/local-docker-deployments.yaml: subprocess platform wired with a docker deployments executor (pull_images enabled, tightened reconcile loop). Signed-off-by: Ben McCown --- e2e/configs/local-docker-deployments.yaml | 55 +++ e2e/deployments_helpers.py | 516 ++++++++++++++++++++++ e2e/test_nemo_deployments_docker.py | 132 ++++++ e2e/test_nemo_deployments_k8s.py | 88 ++++ 4 files changed, 791 insertions(+) create mode 100644 e2e/configs/local-docker-deployments.yaml create mode 100644 e2e/deployments_helpers.py create mode 100644 e2e/test_nemo_deployments_docker.py create mode 100644 e2e/test_nemo_deployments_k8s.py diff --git a/e2e/configs/local-docker-deployments.yaml b/e2e/configs/local-docker-deployments.yaml new file mode 100644 index 0000000000..1277dfd554 --- /dev/null +++ b/e2e/configs/local-docker-deployments.yaml @@ -0,0 +1,55 @@ +# E2E config for the nemo-deployments plugin against a real Docker daemon. +# +# The platform runs as a normal local process (subprocess harness backend) with +# BOTH the deployments service and its reconcile controller enabled (the harness +# runs `nemo services run --service-group all --controller-group all`). A single +# Docker executor is registered so DeploymentConfig/Deployment/Volume entities +# created through the deployments API are reconciled into real Docker containers +# and volumes on the host daemon. +# +# Unlike local-docker-agents.yaml, this config leaves pull_images enabled: the +# deployment tests deploy small public images (alpine / nginx) that are not +# guaranteed to be present locally, so the executor must pull them on demand. + +platform: + runtime: "docker" + base_url: "http://0.0.0.0:8080" + +service: {} + +auth: + enabled: false + allow_unsigned_jwt: true + policy_decision_point_provider: embedded + policy_decision_point_base_url: "http://localhost:8080" + policy_data_refresh_interval: 2 + bundle_cache_seconds: 15 + admin_email: "admin@example.com" + +entities: {} + +deployments: + default_executor: local-docker + executors: + - name: local-docker + backend: docker + config: + pull_images: true + port_range_start: 9200 + port_range_end: 9300 + controller: + # Tighten the reconcile loop so deployment/volume state converges quickly in + # the test window; the defaults are tuned for production, not test latency. + interval_seconds: 2 + orphan_cleanup_interval_seconds: 15 + # Fail STARTING deployments that never converge within a few minutes rather + # than letting a test hang until the outer wait-loop timeout. + starting_timeout_seconds: 240 + +secrets: + allow_key_creation: true + +files: + default_storage_config: + type: local + path: .tmp/e2e/files diff --git a/e2e/deployments_helpers.py b/e2e/deployments_helpers.py new file mode 100644 index 0000000000..3cbf039bde --- /dev/null +++ b/e2e/deployments_helpers.py @@ -0,0 +1,516 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared helpers for nemo-deployments plugin e2e tests. + +Both the Docker (``test_nemo_deployments_docker.py``) and Kubernetes +(``test_nemo_deployments_k8s.py``) modules drive the *deployments plugin's own* +public API end to end — DeploymentConfig / Deployment / Volume CRUD plus the +reconcile controller that turns those entities into real backend resources +(Docker containers+volumes, or Kubernetes Job/Deployment+Service+PVC). + +The chain they prove:: + + sdk._client POST /apis/deployments/v2/.../deployment-configs (template) + sdk._client POST /apis/deployments/v2/.../volumes (optional PVC/volume) + sdk._client POST /apis/deployments/v2/.../deployments (desired state) + -> deployments reconcile controller + -> executor backend (docker | k8s) creates the real workload + -> Deployment.status converges (READY for services, SUCCEEDED for jobs) + +Unlike ``e2e/agents_deploy_helpers.py`` (which goes through the higher-level +``sdk.agents`` surface), the deployments plugin is not exposed on the typed SDK, +so this module talks to the REST API directly via ``sdk._client``. The +per-backend modules own only what genuinely differs: pytest markers, the backend +key passed to volume/deployment ``backend_config``, and any best-effort reaping +of leaked backend resources. +""" + +from __future__ import annotations + +import os +import time +import uuid +from collections.abc import Callable +from typing import Any + +import httpx +import pytest +from nemo_platform import NeMoPlatform + +# Small, widely-cached public images used by the deployment workloads. ``alpine`` +# runs a one-shot job (restart_policy=Never -> SUCCEEDED); ``nginx`` runs a +# long-lived service (restart_policy=Always -> READY with an endpoint). +# +# These default to fully-qualified ``docker.io/library/...`` refs but are +# env-overridable, mirroring the ``POSTGRES_IMAGE`` / ``BUSYBOX_IMAGE`` knobs the +# kind e2e Helm install already exposes (see +# ``.github/actions/setup-kind-cluster/action.yaml`` and +# ``e2e/k8s/scripts/install_helm_e2e.sh``). The workspace has no transparent +# DockerHub pull-through cache today, so if one is ever introduced (it would +# require the mirror registry to be named explicitly in the ref), CI can point +# these at it without a code change — exactly as it can for postgres/busybox. +ALPINE_IMAGE = os.environ.get("NMP_E2E_DEPLOYMENTS_ALPINE_IMAGE", "docker.io/library/alpine:3.20") +NGINX_IMAGE = os.environ.get("NMP_E2E_DEPLOYMENTS_NGINX_IMAGE", "docker.io/library/nginx:alpine") + +# Terminal deployment statuses (the reconciler will not transition out of these). +_TERMINAL_DEPLOYMENT_STATUSES = frozenset({"SUCCEEDED", "FAILED", "LOST"}) + + +def unique_name(prefix: str) -> str: + return f"e2e-{prefix}-{uuid.uuid4().hex[:8]}" + + +def _base(workspace: str) -> str: + return f"/apis/deployments/v2/workspaces/{workspace}" + + +# ---- Raw API wrappers ------------------------------------------------------ + + +def create_deployment_config( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + containers: list[dict[str, Any]], + restart_policy: str = "Always", + volume_mounts: list[dict[str, Any]] | None = None, + config_files: list[dict[str, Any]] | None = None, + backend_config: dict[str, Any] | None = None, +) -> dict[str, Any]: + body: dict[str, Any] = { + "name": name, + "containers": containers, + "restart_policy": restart_policy, + } + if volume_mounts is not None: + body["volume_mounts"] = volume_mounts + if config_files is not None: + body["config_files"] = config_files + if backend_config is not None: + body["backend_config"] = backend_config + response = sdk._client.post(f"{_base(workspace)}/deployment-configs", json=body) + response.raise_for_status() + return response.json() + + +def create_volume( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + size: str = "1Gi", + access_modes: list[str] | None = None, + backend_config: dict[str, Any] | None = None, +) -> dict[str, Any]: + body: dict[str, Any] = {"name": name, "size": size} + if access_modes is not None: + body["access_modes"] = access_modes + if backend_config is not None: + body["backend_config"] = backend_config + response = sdk._client.post(f"{_base(workspace)}/volumes", json=body) + response.raise_for_status() + return response.json() + + +def create_deployment( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + deployment_config: str, + desired_state: str = "READY", + executor: str | None = None, + prerequisites: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + body: dict[str, Any] = { + "name": name, + "deployment_config": deployment_config, + "desired_state": desired_state, + } + if executor is not None: + body["executor"] = executor + if prerequisites is not None: + body["prerequisites"] = prerequisites + response = sdk._client.post(f"{_base(workspace)}/deployments", json=body) + response.raise_for_status() + return response.json() + + +def get_deployment(sdk: NeMoPlatform, *, workspace: str, name: str) -> dict[str, Any]: + response = sdk._client.get(f"{_base(workspace)}/deployments/{name}") + response.raise_for_status() + return response.json() + + +def get_volume(sdk: NeMoPlatform, *, workspace: str, name: str) -> dict[str, Any]: + response = sdk._client.get(f"{_base(workspace)}/volumes/{name}") + response.raise_for_status() + return response.json() + + +def list_deployments(sdk: NeMoPlatform, *, workspace: str) -> list[dict[str, Any]]: + response = sdk._client.get(f"{_base(workspace)}/deployments", params={"page_size": 100}) + response.raise_for_status() + data = response.json().get("data", []) + assert isinstance(data, list) + return data + + +def delete_deployment_if_exists(sdk: NeMoPlatform, *, workspace: str, name: str) -> None: + try: + response = sdk._client.delete(f"{_base(workspace)}/deployments/{name}") + response.raise_for_status() + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + raise + + +def delete_volume_if_exists(sdk: NeMoPlatform, *, workspace: str, name: str) -> None: + try: + response = sdk._client.delete(f"{_base(workspace)}/volumes/{name}") + response.raise_for_status() + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + raise + + +def delete_deployment_config_if_exists(sdk: NeMoPlatform, *, workspace: str, name: str) -> None: + try: + response = sdk._client.delete(f"{_base(workspace)}/deployment-configs/{name}") + response.raise_for_status() + except httpx.HTTPStatusError as exc: + if exc.response.status_code != 404: + raise + + +# ---- Wait helpers ---------------------------------------------------------- + + +def wait_for_deployment_status( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + target_statuses: tuple[str, ...], + timeout_seconds: float = 240, +) -> dict[str, Any]: + """Poll a deployment until it reaches one of ``target_statuses``. + + Fails fast if the deployment lands in a terminal status that was not one of + the requested targets (e.g. FAILED while waiting for READY), surfacing the + ``status_message``/``error_details`` to make debugging tractable. + """ + deadline = time.monotonic() + timeout_seconds + last: dict[str, Any] | None = None + while time.monotonic() < deadline: + deployment = get_deployment(sdk, workspace=workspace, name=name) + last = deployment + status = deployment["status"] + if status in target_statuses: + return deployment + if status in _TERMINAL_DEPLOYMENT_STATUSES and status not in target_statuses: + pytest.fail( + f"Deployment {name!r} reached unexpected terminal status {status!r} " + f"while waiting for {target_statuses}: " + f"message={deployment.get('status_message')!r} " + f"error_details={deployment.get('error_details')!r}" + ) + time.sleep(2) + pytest.fail(f"Deployment {name!r} did not reach {target_statuses} within {timeout_seconds}s: {last}") + + +def wait_for_volume_status( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + target_statuses: tuple[str, ...], + timeout_seconds: float = 120, +) -> dict[str, Any]: + deadline = time.monotonic() + timeout_seconds + last: dict[str, Any] | None = None + while time.monotonic() < deadline: + volume = get_volume(sdk, workspace=workspace, name=name) + last = volume + if volume["status"] in target_statuses: + return volume + if volume["status"] == "FAILED" and "FAILED" not in target_statuses: + pytest.fail(f"Volume {name!r} FAILED while waiting for {target_statuses}: {volume.get('status_message')!r}") + time.sleep(2) + pytest.fail(f"Volume {name!r} did not reach {target_statuses} within {timeout_seconds}s: {last}") + + +def wait_for_deployment_deleted( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + timeout_seconds: float = 120, +) -> None: + deadline = time.monotonic() + timeout_seconds + last_status: str | None = None + while time.monotonic() < deadline: + try: + deployment = get_deployment(sdk, workspace=workspace, name=name) + last_status = deployment.get("status") + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return + raise + time.sleep(2) + pytest.fail(f"Deployment {name!r} was not deleted within {timeout_seconds}s; last status={last_status!r}") + + +def wait_for_volume_deleted( + sdk: NeMoPlatform, + *, + workspace: str, + name: str, + timeout_seconds: float = 120, +) -> None: + deadline = time.monotonic() + timeout_seconds + last_status: str | None = None + while time.monotonic() < deadline: + try: + volume = get_volume(sdk, workspace=workspace, name=name) + last_status = volume.get("status") + except httpx.HTTPStatusError as exc: + if exc.response.status_code == 404: + return + raise + time.sleep(2) + pytest.fail(f"Volume {name!r} was not deleted within {timeout_seconds}s; last status={last_status!r}") + + +# ---- Backend-agnostic scenario cores --------------------------------------- + + +def run_service_deployment_lifecycle( + sdk: NeMoPlatform, + *, + workspace: str, + backend_key: str, + deployment_backend_config: dict[str, Any] | None = None, + running_timeout_seconds: float = 240, + reap_backend_resources: Callable[[str], None] | None = None, +) -> None: + """Deploy a long-lived nginx service and assert it reconciles to READY. + + Proves the full create-template -> create-deployment -> reconcile -> READY + chain for a ``restart_policy=Always`` service, including that the backend + surfaces a routable endpoint. Cleans up the deployment and config (best + effort, isolated steps) and, on backends that support it, reaps any leaked + workload via ``reap_backend_resources``. + + ``backend_key`` (``"docker"`` / ``"k8s"``) only affects the optional + ``deployment_backend_config`` the caller passes through; the reconcile path + is otherwise identical across backends. + """ + config_name = unique_name("svc-cfg") + deployment_name = unique_name("svc") + + create_deployment_config( + sdk, + workspace=workspace, + name=config_name, + restart_policy="Always", + containers=[ + { + "name": "main", + "image": NGINX_IMAGE, + "ports": [{"containerPort": 80, "protocol": "TCP", "name": "http"}], + } + ], + backend_config=deployment_backend_config, + ) + + try: + created = create_deployment( + sdk, + workspace=workspace, + name=deployment_name, + deployment_config=config_name, + ) + assert created["name"] == deployment_name + assert created["deployment_config"] == config_name + assert created["status"] == "PENDING" + + deployment = wait_for_deployment_status( + sdk, + workspace=workspace, + name=deployment_name, + target_statuses=("READY",), + timeout_seconds=running_timeout_seconds, + ) + # A long-lived service must expose at least one routable endpoint. + endpoints = deployment.get("endpoints") or [] + assert endpoints and endpoints[0]["url"], deployment + + # It must also show up in the workspace listing while active. + listed = {d["name"] for d in list_deployments(sdk, workspace=workspace)} + assert deployment_name in listed + finally: + _safe(delete_deployment_if_exists, sdk, workspace=workspace, name=deployment_name) + _safe(wait_for_deployment_deleted, sdk, workspace=workspace, name=deployment_name) + if reap_backend_resources is not None: + _safe(reap_backend_resources, deployment_name) + _safe(delete_deployment_config_if_exists, sdk, workspace=workspace, name=config_name) + + +def run_job_deployment_lifecycle( + sdk: NeMoPlatform, + *, + workspace: str, + backend_key: str, + running_timeout_seconds: float = 240, + reap_backend_resources: Callable[[str], None] | None = None, +) -> None: + """Deploy a one-shot alpine job and assert it reconciles to SUCCEEDED. + + Proves the ``restart_policy=Never`` path: the workload runs to completion and + the reconciler records the terminal SUCCEEDED status with exit code 0. + """ + config_name = unique_name("job-cfg") + deployment_name = unique_name("job") + + create_deployment_config( + sdk, + workspace=workspace, + name=config_name, + restart_policy="Never", + containers=[ + { + "name": "main", + "image": ALPINE_IMAGE, + "command": ["sh", "-c"], + "args": ["echo hello-from-deployments-e2e"], + } + ], + ) + + try: + create_deployment( + sdk, + workspace=workspace, + name=deployment_name, + deployment_config=config_name, + ) + deployment = wait_for_deployment_status( + sdk, + workspace=workspace, + name=deployment_name, + target_statuses=("SUCCEEDED",), + timeout_seconds=running_timeout_seconds, + ) + assert deployment.get("exit_code") == 0, deployment + finally: + _safe(delete_deployment_if_exists, sdk, workspace=workspace, name=deployment_name) + _safe(wait_for_deployment_deleted, sdk, workspace=workspace, name=deployment_name) + if reap_backend_resources is not None: + _safe(reap_backend_resources, deployment_name) + _safe(delete_deployment_config_if_exists, sdk, workspace=workspace, name=config_name) + + +def run_volume_deployment_round_trip( + sdk: NeMoPlatform, + *, + workspace: str, + backend_key: str, + volume_backend_config: dict[str, Any] | None = None, + mount_path: str = "/data", + running_timeout_seconds: float = 240, + reap_backend_resources: Callable[[str], None] | None = None, +) -> None: + """Prove a volume is provisioned, mounted, written to, and read back. + + 1. Create a Volume and wait for it to reconcile out of PENDING (BOUND on + docker / eagerly-bound storage; may stay PENDING on lazy-binding k8s + storage classes until first consumed — both are acceptable pre-mount). + 2. Deploy a one-shot job whose DeploymentConfig mounts the volume, writes a + sentinel file, then reads it back and asserts the content. If the mount + did not work the ``grep`` fails and the container exits non-zero, so the + deployment lands FAILED and the wait-for-SUCCEEDED fails the test. + 3. Delete the deployment, then the volume (deletion is blocked by referencing + configs, so the config is dropped first in teardown). + """ + config_name = unique_name("vol-cfg") + volume_name = unique_name("vol") + deployment_name = unique_name("vol-job") + sentinel = f"volume-payload-{uuid.uuid4().hex[:8]}" + sentinel_file = f"{mount_path.rstrip('/')}/sentinel.txt" + + create_volume( + sdk, + workspace=workspace, + name=volume_name, + size="1Gi", + access_modes=["ReadWriteOnce"], + backend_config=volume_backend_config, + ) + + # A freshly-created volume must at least leave the initial state; lazy-binding + # k8s storage classes keep it PENDING until first mount, so accept either. + wait_for_volume_status( + sdk, + workspace=workspace, + name=volume_name, + target_statuses=("BOUND", "PENDING"), + timeout_seconds=120, + ) + + create_deployment_config( + sdk, + workspace=workspace, + name=config_name, + restart_policy="Never", + volume_mounts=[{"name": volume_name, "mountPath": mount_path}], + containers=[ + { + "name": "main", + "image": ALPINE_IMAGE, + "command": ["sh", "-c"], + "args": [ + # Write a sentinel to the mounted volume then read it back and + # assert its content, exiting non-zero (=> FAILED) on mismatch. + f"set -e; echo {sentinel} > {sentinel_file}; grep -q {sentinel} {sentinel_file}; " + f"echo mount-verified", + ], + "volumeMounts": [{"name": volume_name, "mountPath": mount_path}], + } + ], + ) + + try: + create_deployment( + sdk, + workspace=workspace, + name=deployment_name, + deployment_config=config_name, + ) + deployment = wait_for_deployment_status( + sdk, + workspace=workspace, + name=deployment_name, + target_statuses=("SUCCEEDED",), + timeout_seconds=running_timeout_seconds, + ) + assert deployment.get("exit_code") == 0, deployment + finally: + _safe(delete_deployment_if_exists, sdk, workspace=workspace, name=deployment_name) + _safe(wait_for_deployment_deleted, sdk, workspace=workspace, name=deployment_name) + if reap_backend_resources is not None: + _safe(reap_backend_resources, deployment_name) + # The volume delete is blocked while a config still mounts it, so the + # config must be dropped before the volume. + _safe(delete_deployment_config_if_exists, sdk, workspace=workspace, name=config_name) + _safe(delete_volume_if_exists, sdk, workspace=workspace, name=volume_name) + _safe(wait_for_volume_deleted, sdk, workspace=workspace, name=volume_name) + + +def _safe(fn: Any, *args: Any, **kwargs: Any) -> None: + try: + fn(*args, **kwargs) + except Exception: + pass diff --git a/e2e/test_nemo_deployments_docker.py b/e2e/test_nemo_deployments_docker.py new file mode 100644 index 0000000000..5b57f3666f --- /dev/null +++ b/e2e/test_nemo_deployments_docker.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""E2E tests for the nemo-deployments plugin against a real Docker daemon. + +Where ``test_nemo_agents_docker.py`` exercises the *agents* surface (which uses +the deployments plugin under the hood for a single opinionated agent container), +this module exercises the deployments plugin's **own** public API directly: +DeploymentConfig / Deployment / Volume CRUD plus the reconcile controller that +turns those entities into real Docker containers and volumes. The +backend-agnostic scenario cores are shared with the Kubernetes variant +(``test_nemo_deployments_k8s.py``) via ``e2e.deployments_helpers``; this module +owns only the docker-specific wiring. + +What it proves — the deployments reconcile chain end to end, on Docker:: + + sdk._client POST /apis/deployments/v2/... (config / volume / deployment) + -> deployments reconcile controller + -> docker executor creates the container / named volume + -> Deployment.status converges (READY for the nginx service, + SUCCEEDED for the alpine job and the volume round-trip job) + +How it runs, and where: + +- The platform runs as a normal local process (subprocess harness) wired with a + nemo-deployments Docker executor (see ``e2e/configs/local-docker-deployments.yaml``). + The harness runs both the deployments service and its reconcile controller. +- The workloads use small public images (``alpine`` / ``nginx``); the executor + pulls them on demand (``pull_images: true`` in the config), so no prebuilt + ``nmp-api`` image is needed here — hence no ``needs_nmp_api_image`` marker. + The image refs are env-overridable (see ``e2e.deployments_helpers``) to match + the ``POSTGRES_IMAGE`` / ``BUSYBOX_IMAGE`` knobs the k8s e2e install exposes, + should a DockerHub mirror ever be introduced. +- ``subprocess_only``: this module drives its own subprocess-harness platform + configured with a docker deployments executor. It must NOT run against an + external cluster (``NMP_BASE_URL`` set), where its ``e2e_config`` / harness are + ignored and no docker executor exists. +- Docker-only workloads run on the host daemon directly. Unlike the agents docker + test, nothing here needs the docker-bridge base-url rewrite (no in-container + callback to the platform), so there is no ``container_base_url_host`` harness + option and no Linux-only skip is strictly required — but a working Docker + daemon is. The module skips cleanly if the daemon is unreachable. +""" + +from __future__ import annotations + +import pytest +from nemo_platform import NeMoPlatform + +from e2e.deployments_helpers import ( + run_job_deployment_lifecycle, + run_service_deployment_lifecycle, + run_volume_deployment_round_trip, +) + +pytestmark = [ + pytest.mark.subprocess_only, + pytest.mark.e2e_config( + "e2e/configs/local-docker-deployments.yaml", + harness={"backend": "subprocess"}, + ), +] + + +def _remove_deployment_container_if_present(deployment_name: str) -> None: + """Best-effort removal of a leaked deployment container after teardown.""" + try: + from docker.errors import NotFound + + import docker + except Exception: + return + try: + client = docker.from_env() + except Exception: + return + # The docker backend names containers after the deployment (hashed identity); + # match loosely so a naming-scheme change does not silently leak containers. + for container in client.containers.list(all=True): + if deployment_name in container.name: + try: + container.remove(force=True) + except NotFound: + pass + except Exception: + pass + + +def _skip_without_docker() -> None: + try: + import docker + except Exception: + pytest.skip("docker SDK not importable") + return + try: + client = docker.from_env() + client.ping() + except Exception as exc: # pragma: no cover - environment dependent + pytest.skip(f"Docker daemon not reachable: {exc}") + + +def test_docker_service_deployment_reaches_ready(sdk: NeMoPlatform, workspace: str) -> None: + """A restart_policy=Always nginx service reconciles to READY with an endpoint.""" + _skip_without_docker() + run_service_deployment_lifecycle( + sdk, + workspace=workspace, + backend_key="docker", + reap_backend_resources=_remove_deployment_container_if_present, + ) + + +def test_docker_job_deployment_reaches_succeeded(sdk: NeMoPlatform, workspace: str) -> None: + """A restart_policy=Never alpine job runs to completion (SUCCEEDED, exit 0).""" + _skip_without_docker() + run_job_deployment_lifecycle( + sdk, + workspace=workspace, + backend_key="docker", + reap_backend_resources=_remove_deployment_container_if_present, + ) + + +def test_docker_volume_is_provisioned_mounted_and_readable(sdk: NeMoPlatform, workspace: str) -> None: + """A named volume is provisioned, mounted into a job, written to, and read back.""" + _skip_without_docker() + run_volume_deployment_round_trip( + sdk, + workspace=workspace, + backend_key="docker", + reap_backend_resources=_remove_deployment_container_if_present, + ) diff --git a/e2e/test_nemo_deployments_k8s.py b/e2e/test_nemo_deployments_k8s.py new file mode 100644 index 0000000000..c39b784974 --- /dev/null +++ b/e2e/test_nemo_deployments_k8s.py @@ -0,0 +1,88 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""E2E tests for the nemo-deployments plugin on Kubernetes. + +The Kubernetes counterpart to ``test_nemo_deployments_docker.py``: it drives the +deployments plugin's own public API (DeploymentConfig / Deployment / Volume) and +asserts the reconcile controller turns those entities into real Kubernetes +workloads — a Deployment+Service for the long-lived nginx service, a Job for the +one-shot alpine workloads, and a PVC for the volume round-trip. The +backend-agnostic scenario cores are shared with the docker variant via +``e2e.deployments_helpers``; this module owns only the k8s-specific wiring. + +What it proves — the deployments reconcile chain end to end, on Kubernetes:: + + sdk._client POST /apis/deployments/v2/... (config / volume / deployment) + -> deployments reconcile controller + -> k8s executor creates the Deployment+Service / Job / PVC + -> Deployment.status converges (READY for the service, SUCCEEDED for jobs) + +How it runs, and where: + +- ``container_only``: this test only runs against an **external cluster** + (``NMP_BASE_URL`` set) — the Kind CPU e2e CI job, whose Helm-deployed platform + is configured with a nemo-deployments ``k8s`` executor (see + ``e2e/k8s/values/kind.yaml``). It is skipped for the subprocess harness (local + / plain e2e job), which has no k8s executor. This is the inverse of the docker + module's ``subprocess_only``. +- The workloads use small public images (``alpine`` / ``nginx``) pulled by the + kind nodes on demand, so — unlike the agents k8s test — this does not depend on + a node-pre-pulled ``nmp-api`` image and carries no ``needs_nmp_api_image`` + marker. Pulling public ``docker.io/library/...`` images at cluster runtime is + the same pattern the kind e2e job already relies on for postgres / busybox / + cloud-provider-kind (there is no pull-through cache configured). The refs are + env-overridable (see ``e2e.deployments_helpers``) for parity with the + ``POSTGRES_IMAGE`` / ``BUSYBOX_IMAGE`` install knobs. +- The workloads land in the executor's namespace (the Helm release namespace, + beside the platform), reachable in-cluster. +- Pod scheduling, PVC binding, and (internet) image pulls can take longer than + the local docker path, so the scenario cores are given a wider timeout. +""" + +from __future__ import annotations + +import pytest +from nemo_platform import NeMoPlatform + +from e2e.deployments_helpers import ( + run_job_deployment_lifecycle, + run_service_deployment_lifecycle, + run_volume_deployment_round_trip, +) + +# Pod scheduling + PVC binding + image pulls in a fresh cluster take longer than +# a local docker container start. +_K8S_TIMEOUT_SECONDS = 420 + +pytestmark = [pytest.mark.container_only] + + +def test_k8s_service_deployment_reaches_ready(sdk: NeMoPlatform, workspace: str) -> None: + """A restart_policy=Always nginx service reconciles to a k8s Deployment+Service (READY).""" + run_service_deployment_lifecycle( + sdk, + workspace=workspace, + backend_key="k8s", + running_timeout_seconds=_K8S_TIMEOUT_SECONDS, + ) + + +def test_k8s_job_deployment_reaches_succeeded(sdk: NeMoPlatform, workspace: str) -> None: + """A restart_policy=Never alpine job reconciles to a k8s Job that completes (SUCCEEDED).""" + run_job_deployment_lifecycle( + sdk, + workspace=workspace, + backend_key="k8s", + running_timeout_seconds=_K8S_TIMEOUT_SECONDS, + ) + + +def test_k8s_volume_is_provisioned_mounted_and_readable(sdk: NeMoPlatform, workspace: str) -> None: + """A PVC is provisioned, mounted into a Job, written to, and read back.""" + run_volume_deployment_round_trip( + sdk, + workspace=workspace, + backend_key="k8s", + running_timeout_seconds=_K8S_TIMEOUT_SECONDS, + ) From 0653d388958328cbd8904eaffa7c4d9207dc5e60 Mon Sep 17 00:00:00 2001 From: Ben McCown Date: Fri, 17 Jul 2026 12:16:55 -0600 Subject: [PATCH 2/3] test(e2e): drop k8s volume round-trip (deadlocks on WaitForFirstConsumer) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Manual testing in a kind cluster surfaced a hard deadlock in the k8s volume round-trip: DeploymentReconciler gates a deployment's create on every mounted Volume already being BOUND (volume_mounts_ready), but kind's default local-path StorageClass binds WaitForFirstConsumer, so the PVC only binds once a consumer pod is scheduled — which never happens while the deployment is gated. The Kind CPU e2e job uses this same storage class, so the test would have hung until timeout in CI. This mirrors a known limitation the k8s reconcile integration test already documents and sidesteps (tests/integration/test_reconcile_k8s.py). - Remove the volume scenario from test_nemo_deployments_k8s.py (keeps service->READY and job->SUCCEEDED, both verified passing against a kind Helm platform). - Keep the volume round-trip in the docker module, where volumes bind eagerly (verified passing); tighten its volume-status wait to require BOUND rather than tolerating PENDING. - Document the storage-binding portability constraint in both the k8s module and the run_volume_deployment_round_trip helper. Signed-off-by: Ben McCown --- e2e/deployments_helpers.py | 21 ++++++++++---- e2e/test_nemo_deployments_k8s.py | 50 +++++++++++++++++--------------- 2 files changed, 41 insertions(+), 30 deletions(-) diff --git a/e2e/deployments_helpers.py b/e2e/deployments_helpers.py index 3cbf039bde..13feb0b2a4 100644 --- a/e2e/deployments_helpers.py +++ b/e2e/deployments_helpers.py @@ -425,15 +425,23 @@ def run_volume_deployment_round_trip( ) -> None: """Prove a volume is provisioned, mounted, written to, and read back. - 1. Create a Volume and wait for it to reconcile out of PENDING (BOUND on - docker / eagerly-bound storage; may stay PENDING on lazy-binding k8s - storage classes until first consumed — both are acceptable pre-mount). + 1. Create a Volume and wait for it to reconcile to BOUND. 2. Deploy a one-shot job whose DeploymentConfig mounts the volume, writes a sentinel file, then reads it back and asserts the content. If the mount did not work the ``grep`` fails and the container exits non-zero, so the deployment lands FAILED and the wait-for-SUCCEEDED fails the test. 3. Delete the deployment, then the volume (deletion is blocked by referencing configs, so the config is dropped first in teardown). + + Backend portability: this requires the volume to reach BOUND *before* the + mounting deployment starts, because ``DeploymentReconciler`` gates deployment + create on all mounted volumes being BOUND (see ``volume_mounts_ready``). That + holds on eagerly-binding storage (the docker backend binds immediately; k8s + ``Immediate``-binding StorageClasses likewise). It does **not** hold on + ``WaitForFirstConsumer`` storage (e.g. kind's default ``local-path``), where + the PVC only binds once a consumer pod is scheduled — a chicken-and-egg with + the reconciler's gate. This helper is therefore used by the docker module + only; see ``test_nemo_deployments_k8s.py`` for why the k8s module omits it. """ config_name = unique_name("vol-cfg") volume_name = unique_name("vol") @@ -450,13 +458,14 @@ def run_volume_deployment_round_trip( backend_config=volume_backend_config, ) - # A freshly-created volume must at least leave the initial state; lazy-binding - # k8s storage classes keep it PENDING until first mount, so accept either. + # The reconciler gates the mounting deployment on the volume being BOUND, and + # this helper only runs on eagerly-binding backends (docker), so require BOUND + # up front rather than tolerating a lingering PENDING. wait_for_volume_status( sdk, workspace=workspace, name=volume_name, - target_statuses=("BOUND", "PENDING"), + target_statuses=("BOUND",), timeout_seconds=120, ) diff --git a/e2e/test_nemo_deployments_k8s.py b/e2e/test_nemo_deployments_k8s.py index c39b784974..5c43a9a400 100644 --- a/e2e/test_nemo_deployments_k8s.py +++ b/e2e/test_nemo_deployments_k8s.py @@ -4,19 +4,32 @@ """E2E tests for the nemo-deployments plugin on Kubernetes. The Kubernetes counterpart to ``test_nemo_deployments_docker.py``: it drives the -deployments plugin's own public API (DeploymentConfig / Deployment / Volume) and -asserts the reconcile controller turns those entities into real Kubernetes -workloads — a Deployment+Service for the long-lived nginx service, a Job for the -one-shot alpine workloads, and a PVC for the volume round-trip. The -backend-agnostic scenario cores are shared with the docker variant via -``e2e.deployments_helpers``; this module owns only the k8s-specific wiring. +deployments plugin's own public API (DeploymentConfig / Deployment) and asserts +the reconcile controller turns those entities into real Kubernetes workloads — a +Deployment+Service for the long-lived nginx service and a Job for the one-shot +alpine workload. The backend-agnostic scenario cores are shared with the docker +variant via ``e2e.deployments_helpers``; this module owns only the k8s-specific +wiring. What it proves — the deployments reconcile chain end to end, on Kubernetes:: - sdk._client POST /apis/deployments/v2/... (config / volume / deployment) + sdk._client POST /apis/deployments/v2/... (config / deployment) -> deployments reconcile controller - -> k8s executor creates the Deployment+Service / Job / PVC - -> Deployment.status converges (READY for the service, SUCCEEDED for jobs) + -> k8s executor creates the Deployment+Service / Job + -> Deployment.status converges (READY for the service, SUCCEEDED for the job) + +No volume round-trip here (unlike the docker module). The reconciler gates a +deployment's create on every mounted Volume already being ``BOUND`` (see +``volume_mounts_ready``), but kind's default ``local-path`` StorageClass — which +the Kind CPU e2e job uses — binds ``WaitForFirstConsumer``: the PVC only binds +once a consuming pod is scheduled, and that pod is never created while the +deployment is gated. That chicken-and-egg only resolves on ``Immediate``-binding +storage (common outside kind, e.g. most cloud block-storage classes), so a +mounted-volume e2e is not portable to the kind CI environment. This mirrors the +existing k8s reconcile integration test, which deliberately omits a PVC mount for +the same reason (see ``plugins/nemo-deployments/tests/integration/test_reconcile_k8s.py``). +The docker module covers the full provision -> mount -> write -> read-back +round-trip, where volumes bind eagerly. How it runs, and where: @@ -36,8 +49,8 @@ ``POSTGRES_IMAGE`` / ``BUSYBOX_IMAGE`` install knobs. - The workloads land in the executor's namespace (the Helm release namespace, beside the platform), reachable in-cluster. -- Pod scheduling, PVC binding, and (internet) image pulls can take longer than - the local docker path, so the scenario cores are given a wider timeout. +- Pod scheduling and (internet) image pulls can take longer than the local + docker path, so the scenario cores are given a wider timeout. """ from __future__ import annotations @@ -48,11 +61,10 @@ from e2e.deployments_helpers import ( run_job_deployment_lifecycle, run_service_deployment_lifecycle, - run_volume_deployment_round_trip, ) -# Pod scheduling + PVC binding + image pulls in a fresh cluster take longer than -# a local docker container start. +# Pod scheduling + image pulls in a fresh cluster take longer than a local docker +# container start. _K8S_TIMEOUT_SECONDS = 420 pytestmark = [pytest.mark.container_only] @@ -76,13 +88,3 @@ def test_k8s_job_deployment_reaches_succeeded(sdk: NeMoPlatform, workspace: str) backend_key="k8s", running_timeout_seconds=_K8S_TIMEOUT_SECONDS, ) - - -def test_k8s_volume_is_provisioned_mounted_and_readable(sdk: NeMoPlatform, workspace: str) -> None: - """A PVC is provisioned, mounted into a Job, written to, and read back.""" - run_volume_deployment_round_trip( - sdk, - workspace=workspace, - backend_key="k8s", - running_timeout_seconds=_K8S_TIMEOUT_SECONDS, - ) From e6004477eb2861a092c84daf1aab473ac2f3069a Mon Sep 17 00:00:00 2001 From: Ben McCown Date: Wed, 22 Jul 2026 10:26:31 -0600 Subject: [PATCH 3/3] test(e2e): wrap deployment resource creation in teardown-protected try Address CodeRabbit review feedback on PR #766: in the shared deployments e2e lifecycle helpers, resource-creation calls sat before the try block, so a failure during setup (e.g. a volume-status poll timing out or a config create erroring) would bypass the finally cleanup and leak the already-created resources. Move all resource creation inside the existing try in the three scenario cores (run_service_deployment_lifecycle, run_job_deployment_lifecycle, run_volume_deployment_round_trip). The finally blocks are already idempotent (_safe + *_if_exists), so cleaning up not-yet-created resources is a no-op. Signed-off-by: Ben McCown --- e2e/deployments_helpers.py | 152 +++++++++++++++++++------------------ 1 file changed, 80 insertions(+), 72 deletions(-) diff --git a/e2e/deployments_helpers.py b/e2e/deployments_helpers.py index 13feb0b2a4..ab221700e7 100644 --- a/e2e/deployments_helpers.py +++ b/e2e/deployments_helpers.py @@ -311,22 +311,24 @@ def run_service_deployment_lifecycle( config_name = unique_name("svc-cfg") deployment_name = unique_name("svc") - create_deployment_config( - sdk, - workspace=workspace, - name=config_name, - restart_policy="Always", - containers=[ - { - "name": "main", - "image": NGINX_IMAGE, - "ports": [{"containerPort": 80, "protocol": "TCP", "name": "http"}], - } - ], - backend_config=deployment_backend_config, - ) - + # Everything that creates a resource lives inside the try so the finally + # cleanup runs even if config creation fails partway. try: + create_deployment_config( + sdk, + workspace=workspace, + name=config_name, + restart_policy="Always", + containers=[ + { + "name": "main", + "image": NGINX_IMAGE, + "ports": [{"containerPort": 80, "protocol": "TCP", "name": "http"}], + } + ], + backend_config=deployment_backend_config, + ) + created = create_deployment( sdk, workspace=workspace, @@ -375,22 +377,24 @@ def run_job_deployment_lifecycle( config_name = unique_name("job-cfg") deployment_name = unique_name("job") - create_deployment_config( - sdk, - workspace=workspace, - name=config_name, - restart_policy="Never", - containers=[ - { - "name": "main", - "image": ALPINE_IMAGE, - "command": ["sh", "-c"], - "args": ["echo hello-from-deployments-e2e"], - } - ], - ) - + # Everything that creates a resource lives inside the try so the finally + # cleanup runs even if config creation fails partway. try: + create_deployment_config( + sdk, + workspace=workspace, + name=config_name, + restart_policy="Never", + containers=[ + { + "name": "main", + "image": ALPINE_IMAGE, + "command": ["sh", "-c"], + "args": ["echo hello-from-deployments-e2e"], + } + ], + ) + create_deployment( sdk, workspace=workspace, @@ -449,49 +453,53 @@ def run_volume_deployment_round_trip( sentinel = f"volume-payload-{uuid.uuid4().hex[:8]}" sentinel_file = f"{mount_path.rstrip('/')}/sentinel.txt" - create_volume( - sdk, - workspace=workspace, - name=volume_name, - size="1Gi", - access_modes=["ReadWriteOnce"], - backend_config=volume_backend_config, - ) - - # The reconciler gates the mounting deployment on the volume being BOUND, and - # this helper only runs on eagerly-binding backends (docker), so require BOUND - # up front rather than tolerating a lingering PENDING. - wait_for_volume_status( - sdk, - workspace=workspace, - name=volume_name, - target_statuses=("BOUND",), - timeout_seconds=120, - ) - - create_deployment_config( - sdk, - workspace=workspace, - name=config_name, - restart_policy="Never", - volume_mounts=[{"name": volume_name, "mountPath": mount_path}], - containers=[ - { - "name": "main", - "image": ALPINE_IMAGE, - "command": ["sh", "-c"], - "args": [ - # Write a sentinel to the mounted volume then read it back and - # assert its content, exiting non-zero (=> FAILED) on mismatch. - f"set -e; echo {sentinel} > {sentinel_file}; grep -q {sentinel} {sentinel_file}; " - f"echo mount-verified", - ], - "volumeMounts": [{"name": volume_name, "mountPath": mount_path}], - } - ], - ) - + # Everything that creates a resource lives inside the try so the finally + # cleanup runs even if volume creation, polling, or config creation fails + # partway (otherwise a created volume/config would leak). try: + create_volume( + sdk, + workspace=workspace, + name=volume_name, + size="1Gi", + access_modes=["ReadWriteOnce"], + backend_config=volume_backend_config, + ) + + # The reconciler gates the mounting deployment on the volume being BOUND, + # and this helper only runs on eagerly-binding backends (docker), so + # require BOUND up front rather than tolerating a lingering PENDING. + wait_for_volume_status( + sdk, + workspace=workspace, + name=volume_name, + target_statuses=("BOUND",), + timeout_seconds=120, + ) + + create_deployment_config( + sdk, + workspace=workspace, + name=config_name, + restart_policy="Never", + volume_mounts=[{"name": volume_name, "mountPath": mount_path}], + containers=[ + { + "name": "main", + "image": ALPINE_IMAGE, + "command": ["sh", "-c"], + "args": [ + # Write a sentinel to the mounted volume then read it back + # and assert its content, exiting non-zero (=> FAILED) on + # mismatch. + f"set -e; echo {sentinel} > {sentinel_file}; grep -q {sentinel} {sentinel_file}; " + f"echo mount-verified", + ], + "volumeMounts": [{"name": volume_name, "mountPath": mount_path}], + } + ], + ) + create_deployment( sdk, workspace=workspace,