diff --git a/plugins/nemo-deployments/pyproject.toml b/plugins/nemo-deployments/pyproject.toml index 22698169db..c0951f8db1 100644 --- a/plugins/nemo-deployments/pyproject.toml +++ b/plugins/nemo-deployments/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ [project.optional-dependencies] docker = ["docker>=7.0"] +k8s = ["kubernetes>=30.1.0"] [project.entry-points."nemo.services"] deployments = "nemo_deployments_plugin.service:DeploymentsService" @@ -33,7 +34,7 @@ nemo-platform = { workspace = true } nemo-platform-plugin = { workspace = true } [dependency-groups] -dev = ["pytest>=8.3.4", "pytest-asyncio>=0.25.3", "httpx>=0.27", "fastapi>=0.115", "docker>=7.0"] +dev = ["pytest>=8.3.4", "pytest-asyncio>=0.25.3", "httpx>=0.27", "fastapi>=0.115", "docker>=7.0", "kubernetes>=30.1.0"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py index ed91565423..d67994a70b 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py @@ -31,7 +31,15 @@ validate_config_for_docker, ) from nemo_deployments_plugin.backends.docker.gpu import GPUAllocationError, get_shared_gpu_pool -from nemo_deployments_plugin.backends.docker.labels import ( +from nemo_deployments_plugin.backends.docker.ports import find_available_port +from nemo_deployments_plugin.backends.docker.probes import check_readiness_probe, host_url_for_port +from nemo_deployments_plugin.backends.docker.status import ( + LOG_MAX_CHARS, + map_docker_state_to_starting, + map_exited_status, + missing_container_status, +) +from nemo_deployments_plugin.backends.labels import ( BACKOFF_LIMIT_LABEL, CONFIG_NAME_LABEL, DEPLOYMENT_NAME_LABEL, @@ -43,14 +51,6 @@ deployment_key, managed_by_filter, ) -from nemo_deployments_plugin.backends.docker.ports import find_available_port -from nemo_deployments_plugin.backends.docker.probes import check_readiness_probe, host_url_for_port -from nemo_deployments_plugin.backends.docker.status import ( - LOG_MAX_CHARS, - map_docker_state_to_starting, - map_exited_status, - missing_container_status, -) from nemo_deployments_plugin.constants import MANAGED_BY_LABEL from nemo_deployments_plugin.entities import Container, Deployment, DeploymentConfig from nemo_deployments_plugin.types import Endpoint, RestartPolicy diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/containers.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/containers.py index 55f30055f3..00b50cd6e4 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/containers.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/containers.py @@ -7,7 +7,7 @@ from typing import Any -from nemo_deployments_plugin.backends.docker.labels import docker_volume_name +from nemo_deployments_plugin.backends.labels import docker_volume_name from nemo_deployments_plugin.entities import Container, DeploymentConfig, DockerDeploymentConfig, VolumeMount from nemo_deployments_plugin.types import RestartPolicy diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/gpu.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/gpu.py index ab1a69bcc6..a1807600f8 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/gpu.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/gpu.py @@ -131,7 +131,7 @@ def parse_gpu_device_ids(device_requests: list[Any] | None) -> list[int]: def discover_managed_gpu_allocations(client: docker.DockerClient) -> dict[str, list[int]]: """Return workload_id -> GPU IDs for running deployment-managed containers.""" - from nemo_deployments_plugin.backends.docker.labels import ( + from nemo_deployments_plugin.backends.labels import ( DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL, MANAGED_BY_KEY, diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/ports.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/ports.py index cc9239b7f8..d9516a1f4f 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/ports.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/ports.py @@ -11,7 +11,7 @@ import socket from typing import TYPE_CHECKING -from nemo_deployments_plugin.backends.docker.labels import managed_by_filter +from nemo_deployments_plugin.backends.labels import managed_by_filter import docker diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/volumes.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/volumes.py index 35047e012c..3df7a2b1fe 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/volumes.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/volumes.py @@ -10,7 +10,7 @@ from typing import Any from nemo_deployments_plugin.backends.base import VolumeStatusUpdate -from nemo_deployments_plugin.backends.docker.labels import docker_volume_name, volume_identity_labels +from nemo_deployments_plugin.backends.labels import docker_volume_name, volume_identity_labels import docker diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py new file mode 100644 index 0000000000..acafd58274 --- /dev/null +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Kubernetes substrate backend for the deployments plugin (scaffold).""" + +from __future__ import annotations + +import logging +from typing import Any + +from nemo_deployments_plugin.backends.base import ( + BackendStatusUpdate, + DeploymentBackend, + LogResult, + VolumeStatusUpdate, +) +from nemo_deployments_plugin.backends.k8s.client import KubernetesClients +from nemo_deployments_plugin.backends.k8s.config import K8sExecutorConfig + +logger = logging.getLogger(__name__) + +_K8S_INSTALL_HINT = ( + "kubernetes package is required for K8sDeploymentBackend. " + "Install with: uv sync --package nemo-deployments-plugin --extra k8s" +) + + +class K8sDeploymentBackend(DeploymentBackend): + """Manage deployments and volumes as native Kubernetes objects. + + Lifecycle methods not yet implemented raise ``NotImplementedError`` (not ``...``) so + accidental calls fail loudly during phased rollout; ``...`` is for ``@abstractmethod`` + stubs on the ABC itself. + """ + + _clients: KubernetesClients + + def init(self) -> None: + try: + import kubernetes # noqa: F401 + except ImportError as exc: + raise RuntimeError(_K8S_INSTALL_HINT) from exc + + self._executor_config = K8sExecutorConfig.model_validate(self._config) + self._clients = KubernetesClients( + kubeconfig_path=self._executor_config.kubeconfig_path, + request_timeout=self._executor_config.request_timeout, + ) + logger.debug( + "K8sDeploymentBackend initialized (default_namespace=%s)", + self._executor_config.default_namespace, + ) + + def shutdown(self) -> None: + if hasattr(self, "_clients"): + self._clients.close() + + @property + def executor_config(self) -> K8sExecutorConfig: + return self._executor_config + + @property + def clients(self) -> KubernetesClients: + return self._clients + + async def create_deployment( + self, + *, + workspace: str, + name: str, + config_name: str, + labels: dict[str, str], + backend_config: dict[str, Any], + ) -> BackendStatusUpdate: + raise NotImplementedError("K8s create_deployment is implemented in a later phase.") + + async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate: + raise NotImplementedError("K8s read_status is implemented in a later phase.") + + async def delete_deployment(self, workspace: str, name: str) -> BackendStatusUpdate: + raise NotImplementedError("K8s delete_deployment is implemented in a later phase.") + + async def list_managed_deployment_names(self) -> list[str]: + raise NotImplementedError("K8s list_managed_deployment_names is implemented in a later phase.") + + async def get_logs( + self, + *, + workspace: str, + name: str, + tail: int = 100, + ) -> LogResult: + raise NotImplementedError("K8s get_logs is implemented in a later phase.") + + async def create_volume( + self, + *, + workspace: str, + name: str, + size: str, + access_modes: list[str], + backend_config: dict[str, Any], + ) -> VolumeStatusUpdate: + raise NotImplementedError("K8s create_volume is implemented in a later phase.") + + async def read_volume_status(self, *, workspace: str, name: str) -> VolumeStatusUpdate: + raise NotImplementedError("K8s read_volume_status is implemented in a later phase.") + + async def delete_volume(self, workspace: str, name: str) -> VolumeStatusUpdate: + raise NotImplementedError("K8s delete_volume is implemented in a later phase.") diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py new file mode 100644 index 0000000000..0d6ac897d4 --- /dev/null +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Kubernetes client bootstrap for the deployments plugin. + +Copied from the jobs service pattern; tagged for future extraction to a shared substrate lib. + +Imports are centralized in ``_kubernetes_modules()`` rather than hoisted to module scope so +``registry`` can load without requiring the optional ``kubernetes`` package until a k8s +executor is actually constructed. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from kubernetes.client import ApiClient, AppsV1Api, BatchV1Api, CoreV1Api + +logger = logging.getLogger(__name__) + +_kubernetes_modules_cache: tuple[Any, Any] | None = None + + +def _kubernetes_modules() -> tuple[Any, Any]: + """Return ``(kubernetes.client, kubernetes.config)``, importing on first use.""" + global _kubernetes_modules_cache + if _kubernetes_modules_cache is None: + from kubernetes import client, config + + _kubernetes_modules_cache = (client, config) + return _kubernetes_modules_cache + + +def build_api_client(*, kubeconfig_path: str | None = None) -> ApiClient: + """Create an ``ApiClient`` for the given kubeconfig (in-cluster when path is unset).""" + client, config = _kubernetes_modules() + configuration = client.Configuration() + if kubeconfig_path: + config.load_kube_config(config_file=kubeconfig_path, client_configuration=configuration) + else: + try: + config.load_incluster_config(client_configuration=configuration) + except config.ConfigException: + config.load_kube_config(client_configuration=configuration) + return client.ApiClient(configuration) + + +class KubernetesClients: + """Lazy Kubernetes API clients with per-instance kubeconfig and request timeout.""" + + def __init__(self, *, kubeconfig_path: str | None = None, request_timeout: int = 60) -> None: + self._kubeconfig_path = kubeconfig_path + self._request_timeout = request_timeout + self._api_client: ApiClient | None = None + self._core_v1: CoreV1Api | None = None + self._apps_v1: AppsV1Api | None = None + self._batch_v1: BatchV1Api | None = None + + @property + def request_timeout(self) -> int: + """Per-request timeout (seconds) for Kubernetes API calls in later phases.""" + return self._request_timeout + + def _api(self) -> ApiClient: + if self._api_client is None: + self._api_client = build_api_client(kubeconfig_path=self._kubeconfig_path) + logger.debug( + "Kubernetes ApiClient created (kubeconfig_path=%s, request_timeout=%s)", + self._kubeconfig_path, + self._request_timeout, + ) + return self._api_client + + @property + def core_v1(self) -> CoreV1Api: + if self._core_v1 is None: + client, _ = _kubernetes_modules() + self._core_v1 = client.CoreV1Api(self._api()) + return self._core_v1 + + @property + def apps_v1(self) -> AppsV1Api: + if self._apps_v1 is None: + client, _ = _kubernetes_modules() + self._apps_v1 = client.AppsV1Api(self._api()) + return self._apps_v1 + + @property + def batch_v1(self) -> BatchV1Api: + if self._batch_v1 is None: + client, _ = _kubernetes_modules() + self._batch_v1 = client.BatchV1Api(self._api()) + return self._batch_v1 + + def close(self) -> None: + """Release the underlying ``ApiClient`` connection pool, if created. + + ``CoreV1Api`` / ``AppsV1Api`` / ``BatchV1Api`` share the same ``ApiClient`` instance; + closing it invalidates the cached API wrappers (reset below). + """ + if self._api_client is not None: + self._api_client.close() + self._api_client = None + self._core_v1 = None + self._apps_v1 = None + self._batch_v1 = None diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py new file mode 100644 index 0000000000..61d2abb0d0 --- /dev/null +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Executor-level Kubernetes backend configuration.""" + +from __future__ import annotations + +import re + +from pydantic import BaseModel, Field, field_validator + +_DNS_LABEL_PATTERN = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") + + +class K8sExecutorConfig(BaseModel): + """Knobs for a named k8s executor instance (not entity backend_config).""" + + kubeconfig_path: str | None = Field( + default=None, + description="Path to kubeconfig file. When unset, uses in-cluster config or default kubeconfig.", + ) + default_namespace: str = Field( + default="default", + min_length=1, + max_length=63, + description="Namespace for resources when entity backend_config.k8s.namespace is unset.", + ) + request_timeout: int = Field( + default=60, + ge=1, + description="Kubernetes API client timeout in seconds.", + ) + + @field_validator("default_namespace") + @classmethod + def _validate_default_namespace(cls, value: str) -> str: + if not _DNS_LABEL_PATTERN.fullmatch(value): + raise ValueError("default_namespace must be a lowercase DNS-1123 label (alphanumeric, interior hyphens)") + return value diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py similarity index 59% rename from plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py rename to plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py index 5254d2bdb5..430b0835c4 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py @@ -1,17 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Docker resource naming and identity labels for orphan cleanup. +"""Shared deployment/volume identity labels and substrate-safe resource naming. -Resource names are derived from a human-readable prefix plus a deterministic -8-character hash. The hash is computed from ``{workspace}/{name}`` (see -``deployment_key``), not from the hyphen-joined display string, so ambiguous -pairs like ``("foo", "bar-baz")`` and ``("foo-bar", "baz")`` produce distinct -names even when their joined prefixes collide. +Naming uses ``nemo_platform_plugin.k8s_naming`` (plugins cannot import ``nmp_common``). +Identity labels drive orphan cleanup and idempotency across docker and k8s backends. -Naming logic is shared via ``nemo_platform_plugin.k8s_naming`` (plugins cannot -import ``nmp_common``). Orphan cleanup and idempotency rely on identity labels, -not container names alone. +Label domain is ``nemo.nvidia.com/*`` (deployments plugin scope). Core services such as +jobs and models use ``nmp.nvidia.com/*`` today; converging prefixes is out of scope for 757. + +Deployment and volume resources use separate workspace label keys +(``deployment-workspace`` vs ``volume-workspace``) so list/watch queries can target one +resource kind without ambiguous selectors, even though the workspace value is the same string. """ from nemo_deployments_plugin.constants import MANAGED_BY_LABEL @@ -24,6 +24,7 @@ CONFIG_NAME_LABEL = "nemo.nvidia.com/deployment-config" VOLUME_WORKSPACE_LABEL = "nemo.nvidia.com/volume-workspace" VOLUME_NAME_LABEL = "nemo.nvidia.com/volume-name" +BACKOFF_LIMIT_LABEL = "nemo.nvidia.com/backoff-limit" def deployment_key(workspace: str, name: str) -> str: @@ -33,23 +34,30 @@ def deployment_key(workspace: str, name: str) -> str: def container_name(workspace: str, deployment_name: str) -> str: """Docker container name for a deployment (``dep-`` prefix, hashed identity).""" + return k8s_deployment_resource_name(workspace, deployment_name) + + +def docker_volume_name(workspace: str, volume_name: str) -> str: + """Docker volume name for a deployment volume (``dep-vol-`` prefix, hashed identity).""" + return k8s_volume_resource_name(workspace, volume_name) + + +def k8s_deployment_resource_name(workspace: str, deployment_name: str) -> str: + """Kubernetes resource name for a deployment (Deployment, Job, Service, etc.).""" return k8s_safe_name( f"dep-{workspace}-{deployment_name}", hash_input=deployment_key(workspace, deployment_name), ) -def docker_volume_name(workspace: str, volume_name: str) -> str: - """Docker volume name for a deployment volume (``dep-vol-`` prefix, hashed identity).""" +def k8s_volume_resource_name(workspace: str, volume_name: str) -> str: + """Kubernetes PVC name for a deployment volume.""" return k8s_safe_name( f"dep-vol-{workspace}-{volume_name}", hash_input=deployment_key(workspace, volume_name), ) -BACKOFF_LIMIT_LABEL = "nemo.nvidia.com/backoff-limit" - - def deployment_identity_labels( workspace: str, name: str, @@ -58,6 +66,7 @@ def deployment_identity_labels( config_name: str, backoff_limit: int = 6, ) -> dict[str, str]: + """Return identity labels attached to deployment backend resources.""" return { MANAGED_BY_KEY: MANAGED_BY_LABEL, DEPLOYMENT_WORKSPACE_LABEL: workspace, @@ -69,6 +78,7 @@ def deployment_identity_labels( def volume_identity_labels(workspace: str, name: str) -> dict[str, str]: + """Return identity labels attached to volume backend resources.""" return { MANAGED_BY_KEY: MANAGED_BY_LABEL, VOLUME_WORKSPACE_LABEL: workspace, @@ -76,5 +86,11 @@ def volume_identity_labels(workspace: str, name: str) -> dict[str, str]: } -def managed_by_filter() -> dict[str, str | bool]: +def managed_by_filter() -> dict[str, str]: + """Return a Docker SDK filter dict for plugin-managed resources.""" return {"label": f"{MANAGED_BY_KEY}={MANAGED_BY_LABEL}"} + + +def managed_by_label_selector() -> str: + """Kubernetes label selector for plugin-managed resources.""" + return f"{MANAGED_BY_KEY}={MANAGED_BY_LABEL}" diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/registry.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/registry.py index b7d0a6d34e..5c3fbc66d6 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/registry.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/registry.py @@ -11,12 +11,14 @@ from nemo_deployments_plugin.backends.base import DeploymentBackend from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend +from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend from nemo_platform import AsyncNeMoPlatform logger = logging.getLogger(__name__) BACKEND_CLASSES: dict[str, type[DeploymentBackend]] = { "docker": DockerDeploymentBackend, + "k8s": K8sDeploymentBackend, } diff --git a/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py b/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py index e0ca1154f6..58a4fa0d7a 100644 --- a/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py +++ b/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py @@ -12,7 +12,7 @@ from docker_availability import skip_without_docker from integration_helpers import force_remove_container from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend -from nemo_deployments_plugin.backends.docker.labels import container_name +from nemo_deployments_plugin.backends.labels import container_name from nemo_deployments_plugin.backends.registry import BACKEND_CLASSES from nemo_deployments_plugin.constants import MANAGED_BY_LABEL from nemo_deployments_plugin.entities import ( diff --git a/plugins/nemo-deployments/tests/integration/test_reconcile_docker.py b/plugins/nemo-deployments/tests/integration/test_reconcile_docker.py index bc45edb442..74dc8a678f 100644 --- a/plugins/nemo-deployments/tests/integration/test_reconcile_docker.py +++ b/plugins/nemo-deployments/tests/integration/test_reconcile_docker.py @@ -16,7 +16,7 @@ from docker_availability import skip_without_docker from integration_helpers import force_remove_container from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend -from nemo_deployments_plugin.backends.docker.labels import container_name, docker_volume_name +from nemo_deployments_plugin.backends.labels import container_name, docker_volume_name from nemo_deployments_plugin.backends.registry import BACKEND_CLASSES, ExecutorRegistry from nemo_deployments_plugin.config import ControllerConfig from nemo_deployments_plugin.entities import ( diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py index 2fba410e60..08282f8a43 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py @@ -11,7 +11,7 @@ from backends.docker.docker_helpers import container_attrs, sample_config from docker.errors import APIError, NotFound from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend -from nemo_deployments_plugin.backends.docker.labels import ( +from nemo_deployments_plugin.backends.labels import ( CONFIG_NAME_LABEL, DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL, diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_gpu.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_gpu.py index 91fe6330a6..91a60e43d7 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_gpu.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_gpu.py @@ -14,7 +14,7 @@ get_shared_gpu_pool, parse_gpu_device_ids, ) -from nemo_deployments_plugin.backends.docker.labels import DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL +from nemo_deployments_plugin.backends.labels import DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL from nemo_deployments_plugin.constants import MANAGED_BY_LABEL diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py index 0c68cce0a4..2b1d822359 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py @@ -10,7 +10,7 @@ import pytest from backends.docker.docker_helpers import container_attrs, sample_config from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend -from nemo_deployments_plugin.backends.docker.labels import ( +from nemo_deployments_plugin.backends.labels import ( CONFIG_NAME_LABEL, DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL, diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_labels.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_labels.py index 949a579bf2..9466919130 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_labels.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_labels.py @@ -7,7 +7,7 @@ import re -from nemo_deployments_plugin.backends.docker.labels import ( +from nemo_deployments_plugin.backends.labels import ( CONFIG_NAME_LABEL, DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL, diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/conftest.py b/plugins/nemo-deployments/tests/unit/backends/k8s/conftest.py new file mode 100644 index 0000000000..fab6513555 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/conftest.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared fixtures for k8s backend unit tests.""" + +from __future__ import annotations + +from collections.abc import Iterator +from unittest.mock import MagicMock, patch + +import pytest +from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend + + +@pytest.fixture +def mock_sdk() -> MagicMock: + return MagicMock() + + +@pytest.fixture +def mock_k8s_clients() -> MagicMock: + clients = MagicMock() + clients.core_v1 = MagicMock() + clients.apps_v1 = MagicMock() + clients.batch_v1 = MagicMock() + return clients + + +@pytest.fixture +def k8s_backend(mock_sdk: MagicMock, mock_k8s_clients: MagicMock) -> Iterator[K8sDeploymentBackend]: + with patch("nemo_deployments_plugin.backends.k8s.backend.KubernetesClients", return_value=mock_k8s_clients): + backend = K8sDeploymentBackend( + mock_sdk, + {"default_namespace": "nemo-deployments", "request_timeout": 30}, + ) + yield backend diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py new file mode 100644 index 0000000000..07a2af7493 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py @@ -0,0 +1,39 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from unittest.mock import MagicMock + +import pytest +from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend +from nemo_deployments_plugin.backends.k8s.config import K8sExecutorConfig + + +@pytest.mark.asyncio +async def test_create_deployment_not_implemented_yet(k8s_backend: K8sDeploymentBackend) -> None: + with pytest.raises(NotImplementedError, match="create_deployment"): + await k8s_backend.create_deployment( + workspace="default", + name="srv", + config_name="cfg1", + labels={}, + backend_config={}, + ) + + +def test_executor_config_parsed_from_dict(k8s_backend: K8sDeploymentBackend) -> None: + assert k8s_backend.executor_config.default_namespace == "nemo-deployments" + assert k8s_backend.executor_config.request_timeout == 30 + + +def test_shutdown_closes_kubernetes_clients(k8s_backend: K8sDeploymentBackend) -> None: + mock_clients = MagicMock() + k8s_backend._clients = mock_clients + k8s_backend.shutdown() + mock_clients.close.assert_called_once() + + +def test_default_namespace_rejects_invalid_dns_label() -> None: + with pytest.raises(ValueError, match="default_namespace must be a lowercase DNS-1123 label"): + K8sExecutorConfig(default_namespace="X") diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py new file mode 100644 index 0000000000..c5935ed5c0 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from collections.abc import Iterator +from unittest.mock import MagicMock, patch + +import pytest +from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend +from nemo_deployments_plugin.backends.k8s.client import KubernetesClients, build_api_client +from nemo_deployments_plugin.backends.registry import BACKEND_CLASSES, ExecutorRegistry, ExecutorSpec +from nemo_platform import AsyncNeMoPlatform + + +@pytest.fixture(autouse=True) +def _reset_kubernetes_modules_cache() -> Iterator[None]: + import nemo_deployments_plugin.backends.k8s.client as k8s_client + + k8s_client._kubernetes_modules_cache = None + yield + k8s_client._kubernetes_modules_cache = None + + +def test_k8s_backend_registered() -> None: + assert "k8s" in BACKEND_CLASSES + assert BACKEND_CLASSES["k8s"] is K8sDeploymentBackend + + +def test_executor_registry_accepts_k8s_backend() -> None: + sdk = AsyncNeMoPlatform(base_url="http://localhost:8080") + with patch("nemo_deployments_plugin.backends.k8s.backend.KubernetesClients"): + registry = ExecutorRegistry.from_config( + sdk, + [ExecutorSpec(name="cluster", backend="k8s", config={"default_namespace": "default"})], + ) + backend = registry.resolve("cluster") + assert isinstance(backend, K8sDeploymentBackend) + assert backend.executor_config.default_namespace == "default" + + +def test_k8s_backend_missing_kubernetes_package(monkeypatch: pytest.MonkeyPatch) -> None: + import builtins + + real_import = builtins.__import__ + + def guarded_import( + name: str, + globals: dict[str, object] | None = None, + locals: dict[str, object] | None = None, + fromlist: tuple[str, ...] = (), + level: int = 0, + ) -> object: + if name == "kubernetes": + raise ImportError("no kubernetes") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + with pytest.raises(RuntimeError, match="kubernetes package is required"): + K8sDeploymentBackend(MagicMock(), {}) + + +def test_build_api_client_prefers_in_cluster() -> None: + with ( + patch("kubernetes.config.load_incluster_config") as mock_incluster, + patch("kubernetes.config.load_kube_config") as mock_kube, + patch("kubernetes.client.ApiClient") as mock_api_client, + patch("kubernetes.client.Configuration"), + ): + build_api_client() + mock_incluster.assert_called_once() + mock_kube.assert_not_called() + mock_api_client.assert_called_once() + + +def test_build_api_client_falls_back_to_kubeconfig() -> None: + from kubernetes import config + + with ( + patch("kubernetes.config.load_incluster_config", side_effect=config.ConfigException("not in cluster")), + patch("kubernetes.config.load_kube_config") as mock_kube, + patch("kubernetes.client.ApiClient"), + patch("kubernetes.client.Configuration") as mock_configuration, + ): + build_api_client() + mock_kube.assert_called_once_with(client_configuration=mock_configuration.return_value) + + +def test_build_api_client_honors_explicit_kubeconfig_path() -> None: + with ( + patch("kubernetes.config.load_kube_config") as mock_kube, + patch("kubernetes.config.load_incluster_config") as mock_incluster, + patch("kubernetes.client.ApiClient"), + patch("kubernetes.client.Configuration"), + ): + build_api_client(kubeconfig_path="/tmp/kubeconfig") + mock_kube.assert_called_once() + assert mock_kube.call_args.kwargs["config_file"] == "/tmp/kubeconfig" + mock_incluster.assert_not_called() + + +def test_kubernetes_clients_close_releases_api_client() -> None: + mock_api_client = MagicMock() + with patch("nemo_deployments_plugin.backends.k8s.client.build_api_client", return_value=mock_api_client): + clients = KubernetesClients() + _ = clients.core_v1 + clients.close() + mock_api_client.close.assert_called_once() + assert clients._api_client is None + assert clients._core_v1 is None diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_k8s_registry.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_k8s_registry.py new file mode 100644 index 0000000000..a94919a1b5 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_k8s_registry.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from nemo_deployments_plugin.backends.labels import ( + k8s_deployment_resource_name, + k8s_volume_resource_name, + managed_by_label_selector, +) + + +def test_k8s_resource_names_match_docker_prefixes() -> None: + dep = k8s_deployment_resource_name("foo", "bar-baz") + vol = k8s_volume_resource_name("foo", "data") + assert dep.startswith("dep-") + assert vol.startswith("dep-vol-") + assert len(dep) <= 63 + assert len(vol) <= 63 + + +def test_ambiguous_workspace_name_pairs_get_distinct_k8s_names() -> None: + a = k8s_deployment_resource_name("foo", "bar-baz") + b = k8s_deployment_resource_name("foo-bar", "baz") + assert a != b + + +def test_managed_by_label_selector() -> None: + assert managed_by_label_selector() == "managed-by=nemo-deployments" diff --git a/uv.lock b/uv.lock index 91c87960a8..442bbb9c64 100644 --- a/uv.lock +++ b/uv.lock @@ -4329,12 +4329,16 @@ dependencies = [ docker = [ { name = "docker", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] +k8s = [ + { name = "kubernetes", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] [package.dev-dependencies] dev = [ { name = "docker", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "fastapi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "kubernetes", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pytest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pytest-asyncio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] @@ -4344,17 +4348,19 @@ requires-dist = [ { name = "docker", marker = "extra == 'docker'", specifier = ">=7.0" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", specifier = ">=0.27" }, + { name = "kubernetes", marker = "extra == 'k8s'", specifier = ">=30.1.0" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, { name = "pydantic", specifier = ">=2.10.6" }, ] -provides-extras = ["docker"] +provides-extras = ["docker", "k8s"] [package.metadata.requires-dev] dev = [ { name = "docker", specifier = ">=7.0" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", specifier = ">=0.27" }, + { name = "kubernetes", specifier = ">=30.1.0" }, { name = "pytest", specifier = ">=8.3.4" }, { name = "pytest-asyncio", specifier = ">=0.25.3" }, ]