From 45ad6aee05acdd2c85f2bc690f2821f44b16f4a0 Mon Sep 17 00:00:00 2001 From: Tyler Bray Date: Wed, 1 Jul 2026 08:34:49 -0700 Subject: [PATCH 1/3] feat(deployments): scaffold K8s DeploymentBackend (AIRCORE-757 phase 1) Register K8sDeploymentBackend with executor config, per-instance Kubernetes clients, and shared identity labels. ABC methods stub NotImplementedError until later phases implement PVC, Job, and Deployment+Service paths. Signed-off-by: Tyler Bray --- plugins/nemo-deployments/pyproject.toml | 3 +- .../backends/docker/labels.py | 112 ++++++------------ .../backends/k8s/backend.py | 104 ++++++++++++++++ .../backends/k8s/client.py | 83 +++++++++++++ .../backends/k8s/config.py | 27 +++++ .../backends/labels.py | 86 ++++++++++++++ .../backends/registry.py | 2 + .../tests/unit/backends/k8s/conftest.py | 36 ++++++ .../tests/unit/backends/k8s/test_backend.py | 24 ++++ .../tests/unit/backends/k8s/test_client.py | 102 ++++++++++++++++ .../unit/backends/k8s/test_k8s_registry.py | 29 +++++ uv.lock | 8 +- 12 files changed, 537 insertions(+), 79 deletions(-) create mode 100644 plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py create mode 100644 plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py create mode 100644 plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py create mode 100644 plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py create mode 100644 plugins/nemo-deployments/tests/unit/backends/k8s/conftest.py create mode 100644 plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py create mode 100644 plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py create mode 100644 plugins/nemo-deployments/tests/unit/backends/k8s/test_k8s_registry.py diff --git a/plugins/nemo-deployments/pyproject.toml b/plugins/nemo-deployments/pyproject.toml index 22698169db..c0951f8db1 100644 --- a/plugins/nemo-deployments/pyproject.toml +++ b/plugins/nemo-deployments/pyproject.toml @@ -14,6 +14,7 @@ dependencies = [ [project.optional-dependencies] docker = ["docker>=7.0"] +k8s = ["kubernetes>=30.1.0"] [project.entry-points."nemo.services"] deployments = "nemo_deployments_plugin.service:DeploymentsService" @@ -33,7 +34,7 @@ nemo-platform = { workspace = true } nemo-platform-plugin = { workspace = true } [dependency-groups] -dev = ["pytest>=8.3.4", "pytest-asyncio>=0.25.3", "httpx>=0.27", "fastapi>=0.115", "docker>=7.0"] +dev = ["pytest>=8.3.4", "pytest-asyncio>=0.25.3", "httpx>=0.27", "fastapi>=0.115", "docker>=7.0", "kubernetes>=30.1.0"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py index 5254d2bdb5..a2627250f2 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py @@ -1,80 +1,38 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Docker resource naming and identity labels for orphan cleanup. - -Resource names are derived from a human-readable prefix plus a deterministic -8-character hash. The hash is computed from ``{workspace}/{name}`` (see -``deployment_key``), not from the hyphen-joined display string, so ambiguous -pairs like ``("foo", "bar-baz")`` and ``("foo-bar", "baz")`` produce distinct -names even when their joined prefixes collide. - -Naming logic is shared via ``nemo_platform_plugin.k8s_naming`` (plugins cannot -import ``nmp_common``). Orphan cleanup and idempotency rely on identity labels, -not container names alone. -""" - -from nemo_deployments_plugin.constants import MANAGED_BY_LABEL -from nemo_platform_plugin.k8s_naming import k8s_safe_name, workspace_name_identity - -MANAGED_BY_KEY = "managed-by" -DEPLOYMENT_WORKSPACE_LABEL = "nemo.nvidia.com/deployment-workspace" -DEPLOYMENT_NAME_LABEL = "nemo.nvidia.com/deployment-name" -RESTART_POLICY_LABEL = "nemo.nvidia.com/restart-policy" -CONFIG_NAME_LABEL = "nemo.nvidia.com/deployment-config" -VOLUME_WORKSPACE_LABEL = "nemo.nvidia.com/volume-workspace" -VOLUME_NAME_LABEL = "nemo.nvidia.com/volume-name" - - -def deployment_key(workspace: str, name: str) -> str: - """Return the canonical identity string used for hashing and label keys.""" - return workspace_name_identity(workspace, name) - - -def container_name(workspace: str, deployment_name: str) -> str: - """Docker container name for a deployment (``dep-`` prefix, hashed identity).""" - return k8s_safe_name( - f"dep-{workspace}-{deployment_name}", - hash_input=deployment_key(workspace, deployment_name), - ) - - -def docker_volume_name(workspace: str, volume_name: str) -> str: - """Docker volume name for a deployment volume (``dep-vol-`` prefix, hashed identity).""" - return k8s_safe_name( - f"dep-vol-{workspace}-{volume_name}", - hash_input=deployment_key(workspace, volume_name), - ) - - -BACKOFF_LIMIT_LABEL = "nemo.nvidia.com/backoff-limit" - - -def deployment_identity_labels( - workspace: str, - name: str, - restart_policy: str, - *, - config_name: str, - backoff_limit: int = 6, -) -> dict[str, str]: - return { - MANAGED_BY_KEY: MANAGED_BY_LABEL, - DEPLOYMENT_WORKSPACE_LABEL: workspace, - DEPLOYMENT_NAME_LABEL: name, - RESTART_POLICY_LABEL: restart_policy, - CONFIG_NAME_LABEL: config_name, - BACKOFF_LIMIT_LABEL: str(backoff_limit), - } - - -def volume_identity_labels(workspace: str, name: str) -> dict[str, str]: - return { - MANAGED_BY_KEY: MANAGED_BY_LABEL, - VOLUME_WORKSPACE_LABEL: workspace, - VOLUME_NAME_LABEL: name, - } - - -def managed_by_filter() -> dict[str, str | bool]: - return {"label": f"{MANAGED_BY_KEY}={MANAGED_BY_LABEL}"} +"""Docker resource naming and identity labels (re-exported from shared ``backends.labels``).""" + +from nemo_deployments_plugin.backends.labels import ( + BACKOFF_LIMIT_LABEL, + CONFIG_NAME_LABEL, + DEPLOYMENT_NAME_LABEL, + DEPLOYMENT_WORKSPACE_LABEL, + MANAGED_BY_KEY, + RESTART_POLICY_LABEL, + VOLUME_NAME_LABEL, + VOLUME_WORKSPACE_LABEL, + container_name, + deployment_identity_labels, + deployment_key, + docker_volume_name, + managed_by_filter, + volume_identity_labels, +) + +__all__ = [ + "BACKOFF_LIMIT_LABEL", + "CONFIG_NAME_LABEL", + "DEPLOYMENT_NAME_LABEL", + "DEPLOYMENT_WORKSPACE_LABEL", + "MANAGED_BY_KEY", + "RESTART_POLICY_LABEL", + "VOLUME_NAME_LABEL", + "VOLUME_WORKSPACE_LABEL", + "container_name", + "deployment_identity_labels", + "deployment_key", + "docker_volume_name", + "managed_by_filter", + "volume_identity_labels", +] diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py new file mode 100644 index 0000000000..ffebcd1be9 --- /dev/null +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Kubernetes substrate backend for the deployments plugin (scaffold).""" + +from __future__ import annotations + +import logging +from typing import Any + +from nemo_deployments_plugin.backends.base import ( + BackendStatusUpdate, + DeploymentBackend, + LogResult, + VolumeStatusUpdate, +) +from nemo_deployments_plugin.backends.k8s.client import KubernetesClients +from nemo_deployments_plugin.backends.k8s.config import K8sExecutorConfig + +logger = logging.getLogger(__name__) + +_K8S_INSTALL_HINT = ( + "kubernetes package is required for K8sDeploymentBackend. " + "Install with: uv sync --package nemo-deployments-plugin --extra k8s" +) + + +class K8sDeploymentBackend(DeploymentBackend): + """Manage deployments and volumes as native Kubernetes objects.""" + + _clients: KubernetesClients + + def init(self) -> None: + try: + import kubernetes # noqa: F401 + except ImportError as exc: + raise RuntimeError(_K8S_INSTALL_HINT) from exc + + self._executor_config = K8sExecutorConfig.model_validate(self._config) + self._clients = KubernetesClients( + kubeconfig_path=self._executor_config.kubeconfig_path, + request_timeout=self._executor_config.request_timeout, + ) + logger.debug( + "K8sDeploymentBackend initialized (default_namespace=%s)", + self._executor_config.default_namespace, + ) + + def shutdown(self) -> None: + pass + + @property + def executor_config(self) -> K8sExecutorConfig: + return self._executor_config + + @property + def clients(self) -> KubernetesClients: + return self._clients + + async def create_deployment( + self, + *, + workspace: str, + name: str, + config_name: str, + labels: dict[str, str], + backend_config: dict[str, Any], + ) -> BackendStatusUpdate: + raise NotImplementedError("K8s create_deployment is implemented in a later phase.") + + async def read_status(self, *, workspace: str, name: str) -> BackendStatusUpdate: + raise NotImplementedError("K8s read_status is implemented in a later phase.") + + async def delete_deployment(self, workspace: str, name: str) -> BackendStatusUpdate: + raise NotImplementedError("K8s delete_deployment is implemented in a later phase.") + + async def list_managed_deployment_names(self) -> list[str]: + raise NotImplementedError("K8s list_managed_deployment_names is implemented in a later phase.") + + async def get_logs( + self, + *, + workspace: str, + name: str, + tail: int = 100, + ) -> LogResult: + raise NotImplementedError("K8s get_logs is implemented in a later phase.") + + async def create_volume( + self, + *, + workspace: str, + name: str, + size: str, + access_modes: list[str], + backend_config: dict[str, Any], + ) -> VolumeStatusUpdate: + raise NotImplementedError("K8s create_volume is implemented in a later phase.") + + async def read_volume_status(self, *, workspace: str, name: str) -> VolumeStatusUpdate: + raise NotImplementedError("K8s read_volume_status is implemented in a later phase.") + + async def delete_volume(self, workspace: str, name: str) -> VolumeStatusUpdate: + raise NotImplementedError("K8s delete_volume is implemented in a later phase.") diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py new file mode 100644 index 0000000000..607d160659 --- /dev/null +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Kubernetes client bootstrap for the deployments plugin. + +Copied from the jobs service pattern; tagged for future extraction to a shared substrate lib. +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from kubernetes import client + +logger = logging.getLogger(__name__) + + +def build_api_client(*, kubeconfig_path: str | None = None) -> client.ApiClient: + """Create an ``ApiClient`` for the given kubeconfig (in-cluster when path is unset).""" + from kubernetes import client, config + + configuration = client.Configuration() + if kubeconfig_path: + config.load_kube_config(config_file=kubeconfig_path, client_configuration=configuration) + else: + try: + config.load_incluster_config(client_configuration=configuration) + except config.ConfigException: + config.load_kube_config(client_configuration=configuration) + return client.ApiClient(configuration) + + +class KubernetesClients: + """Lazy Kubernetes API clients with per-instance kubeconfig and request timeout.""" + + def __init__(self, *, kubeconfig_path: str | None = None, request_timeout: int = 60) -> None: + self._kubeconfig_path = kubeconfig_path + self._request_timeout = request_timeout + self._api_client: client.ApiClient | None = None + self._core_v1: client.CoreV1Api | None = None + self._apps_v1: client.AppsV1Api | None = None + self._batch_v1: client.BatchV1Api | None = None + + @property + def request_timeout(self) -> int: + """Per-request timeout (seconds) for Kubernetes API calls in later phases.""" + return self._request_timeout + + def _api(self) -> client.ApiClient: + if self._api_client is None: + self._api_client = build_api_client(kubeconfig_path=self._kubeconfig_path) + logger.debug( + "Kubernetes ApiClient created (kubeconfig_path=%s, request_timeout=%s)", + self._kubeconfig_path, + self._request_timeout, + ) + return self._api_client + + @property + def core_v1(self) -> client.CoreV1Api: + if self._core_v1 is None: + from kubernetes import client + + self._core_v1 = client.CoreV1Api(self._api()) + return self._core_v1 + + @property + def apps_v1(self) -> client.AppsV1Api: + if self._apps_v1 is None: + from kubernetes import client + + self._apps_v1 = client.AppsV1Api(self._api()) + return self._apps_v1 + + @property + def batch_v1(self) -> client.BatchV1Api: + if self._batch_v1 is None: + from kubernetes import client + + self._batch_v1 = client.BatchV1Api(self._api()) + return self._batch_v1 diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py new file mode 100644 index 0000000000..7375a1165e --- /dev/null +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py @@ -0,0 +1,27 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Executor-level Kubernetes backend configuration.""" + +from __future__ import annotations + +from pydantic import BaseModel, Field + + +class K8sExecutorConfig(BaseModel): + """Knobs for a named k8s executor instance (not entity backend_config).""" + + kubeconfig_path: str | None = Field( + default=None, + description="Path to kubeconfig file. When unset, uses in-cluster config or default kubeconfig.", + ) + default_namespace: str = Field( + default="default", + min_length=1, + description="Namespace for resources when entity backend_config.k8s.namespace is unset.", + ) + request_timeout: int = Field( + default=60, + ge=1, + description="Kubernetes API client timeout in seconds.", + ) diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py new file mode 100644 index 0000000000..94a36c5ac7 --- /dev/null +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared deployment/volume identity labels and substrate-safe resource naming. + +Naming uses ``nemo_platform_plugin.k8s_naming`` (plugins cannot import ``nmp_common``). +Identity labels drive orphan cleanup and idempotency across docker and k8s backends. +""" + +from nemo_deployments_plugin.constants import MANAGED_BY_LABEL +from nemo_platform_plugin.k8s_naming import k8s_safe_name, workspace_name_identity + +MANAGED_BY_KEY = "managed-by" +DEPLOYMENT_WORKSPACE_LABEL = "nemo.nvidia.com/deployment-workspace" +DEPLOYMENT_NAME_LABEL = "nemo.nvidia.com/deployment-name" +RESTART_POLICY_LABEL = "nemo.nvidia.com/restart-policy" +CONFIG_NAME_LABEL = "nemo.nvidia.com/deployment-config" +VOLUME_WORKSPACE_LABEL = "nemo.nvidia.com/volume-workspace" +VOLUME_NAME_LABEL = "nemo.nvidia.com/volume-name" +BACKOFF_LIMIT_LABEL = "nemo.nvidia.com/backoff-limit" + + +def deployment_key(workspace: str, name: str) -> str: + """Return the canonical identity string used for hashing and label keys.""" + return workspace_name_identity(workspace, name) + + +def container_name(workspace: str, deployment_name: str) -> str: + """Docker container name for a deployment (``dep-`` prefix, hashed identity).""" + return k8s_deployment_resource_name(workspace, deployment_name) + + +def docker_volume_name(workspace: str, volume_name: str) -> str: + """Docker volume name for a deployment volume (``dep-vol-`` prefix, hashed identity).""" + return k8s_volume_resource_name(workspace, volume_name) + + +def k8s_deployment_resource_name(workspace: str, deployment_name: str) -> str: + """Kubernetes resource name for a deployment (Deployment, Job, Service, etc.).""" + return k8s_safe_name( + f"dep-{workspace}-{deployment_name}", + hash_input=deployment_key(workspace, deployment_name), + ) + + +def k8s_volume_resource_name(workspace: str, volume_name: str) -> str: + """Kubernetes PVC name for a deployment volume.""" + return k8s_safe_name( + f"dep-vol-{workspace}-{volume_name}", + hash_input=deployment_key(workspace, volume_name), + ) + + +def deployment_identity_labels( + workspace: str, + name: str, + restart_policy: str, + *, + config_name: str, + backoff_limit: int = 6, +) -> dict[str, str]: + return { + MANAGED_BY_KEY: MANAGED_BY_LABEL, + DEPLOYMENT_WORKSPACE_LABEL: workspace, + DEPLOYMENT_NAME_LABEL: name, + RESTART_POLICY_LABEL: restart_policy, + CONFIG_NAME_LABEL: config_name, + BACKOFF_LIMIT_LABEL: str(backoff_limit), + } + + +def volume_identity_labels(workspace: str, name: str) -> dict[str, str]: + return { + MANAGED_BY_KEY: MANAGED_BY_LABEL, + VOLUME_WORKSPACE_LABEL: workspace, + VOLUME_NAME_LABEL: name, + } + + +def managed_by_filter() -> dict[str, str | bool]: + return {"label": f"{MANAGED_BY_KEY}={MANAGED_BY_LABEL}"} + + +def managed_by_label_selector() -> str: + """Kubernetes label selector for plugin-managed resources.""" + return f"{MANAGED_BY_KEY}={MANAGED_BY_LABEL}" diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/registry.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/registry.py index b7d0a6d34e..5c3fbc66d6 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/registry.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/registry.py @@ -11,12 +11,14 @@ from nemo_deployments_plugin.backends.base import DeploymentBackend from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend +from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend from nemo_platform import AsyncNeMoPlatform logger = logging.getLogger(__name__) BACKEND_CLASSES: dict[str, type[DeploymentBackend]] = { "docker": DockerDeploymentBackend, + "k8s": K8sDeploymentBackend, } diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/conftest.py b/plugins/nemo-deployments/tests/unit/backends/k8s/conftest.py new file mode 100644 index 0000000000..fab6513555 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/conftest.py @@ -0,0 +1,36 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shared fixtures for k8s backend unit tests.""" + +from __future__ import annotations + +from collections.abc import Iterator +from unittest.mock import MagicMock, patch + +import pytest +from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend + + +@pytest.fixture +def mock_sdk() -> MagicMock: + return MagicMock() + + +@pytest.fixture +def mock_k8s_clients() -> MagicMock: + clients = MagicMock() + clients.core_v1 = MagicMock() + clients.apps_v1 = MagicMock() + clients.batch_v1 = MagicMock() + return clients + + +@pytest.fixture +def k8s_backend(mock_sdk: MagicMock, mock_k8s_clients: MagicMock) -> Iterator[K8sDeploymentBackend]: + with patch("nemo_deployments_plugin.backends.k8s.backend.KubernetesClients", return_value=mock_k8s_clients): + backend = K8sDeploymentBackend( + mock_sdk, + {"default_namespace": "nemo-deployments", "request_timeout": 30}, + ) + yield backend diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py new file mode 100644 index 0000000000..f21e0ea600 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import pytest +from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend + + +@pytest.mark.asyncio +async def test_create_deployment_not_implemented_yet(k8s_backend: K8sDeploymentBackend) -> None: + with pytest.raises(NotImplementedError, match="create_deployment"): + await k8s_backend.create_deployment( + workspace="default", + name="srv", + config_name="cfg1", + labels={}, + backend_config={}, + ) + + +def test_executor_config_parsed_from_dict(k8s_backend: K8sDeploymentBackend) -> None: + assert k8s_backend.executor_config.default_namespace == "nemo-deployments" + assert k8s_backend.executor_config.request_timeout == 30 diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py new file mode 100644 index 0000000000..ef0c8db3a8 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py @@ -0,0 +1,102 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest +from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend +from nemo_deployments_plugin.backends.k8s.client import KubernetesClients, build_api_client +from nemo_deployments_plugin.backends.registry import BACKEND_CLASSES, ExecutorRegistry, ExecutorSpec +from nemo_platform import AsyncNeMoPlatform + + +def test_k8s_backend_registered() -> None: + assert "k8s" in BACKEND_CLASSES + assert BACKEND_CLASSES["k8s"] is K8sDeploymentBackend + + +def test_executor_registry_accepts_k8s_backend() -> None: + sdk = AsyncNeMoPlatform(base_url="http://localhost:8080") + with patch("nemo_deployments_plugin.backends.k8s.backend.KubernetesClients"): + registry = ExecutorRegistry.from_config( + sdk, + [ExecutorSpec(name="cluster", backend="k8s", config={"default_namespace": "default"})], + ) + backend = registry.resolve("cluster") + assert isinstance(backend, K8sDeploymentBackend) + assert backend.executor_config.default_namespace == "default" + + +def test_k8s_backend_missing_kubernetes_package(monkeypatch: pytest.MonkeyPatch) -> None: + import builtins + + real_import = builtins.__import__ + + def guarded_import( + name: str, + globals: dict[str, object] | None = None, + locals: dict[str, object] | None = None, + fromlist: tuple[str, ...] = (), + level: int = 0, + ) -> object: + if name == "kubernetes": + raise ImportError("no kubernetes") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", guarded_import) + with pytest.raises(RuntimeError, match="kubernetes package is required"): + K8sDeploymentBackend(MagicMock(), {}) + + +def test_build_api_client_prefers_in_cluster() -> None: + with ( + patch("kubernetes.config.load_incluster_config") as mock_incluster, + patch("kubernetes.config.load_kube_config") as mock_kube, + patch("kubernetes.client.ApiClient") as mock_api_client, + patch("kubernetes.client.Configuration"), + ): + build_api_client() + mock_incluster.assert_called_once() + mock_kube.assert_not_called() + mock_api_client.assert_called_once() + + +def test_build_api_client_falls_back_to_kubeconfig() -> None: + from kubernetes import config + + with ( + patch("kubernetes.config.load_incluster_config", side_effect=config.ConfigException("not in cluster")), + patch("kubernetes.config.load_kube_config") as mock_kube, + patch("kubernetes.client.ApiClient"), + patch("kubernetes.client.Configuration"), + ): + build_api_client() + mock_kube.assert_called_once_with(client_configuration=mock_kube.call_args.kwargs["client_configuration"]) + + +def test_build_api_client_honors_explicit_kubeconfig_path() -> None: + with ( + patch("kubernetes.config.load_kube_config") as mock_kube, + patch("kubernetes.config.load_incluster_config") as mock_incluster, + patch("kubernetes.client.ApiClient"), + patch("kubernetes.client.Configuration"), + ): + build_api_client(kubeconfig_path="/tmp/kubeconfig") + mock_kube.assert_called_once() + assert mock_kube.call_args.kwargs["config_file"] == "/tmp/kubeconfig" + mock_incluster.assert_not_called() + + +def test_each_kubernetes_clients_instance_uses_its_kubeconfig_path() -> None: + with patch("nemo_deployments_plugin.backends.k8s.client.build_api_client") as mock_build: + mock_build.return_value = MagicMock() + a = KubernetesClients(kubeconfig_path="/a/kube", request_timeout=10) + b = KubernetesClients(kubeconfig_path="/b/kube", request_timeout=20) + _ = a.core_v1 + _ = b.core_v1 + assert mock_build.call_args_list[0].kwargs == {"kubeconfig_path": "/a/kube"} + assert mock_build.call_args_list[1].kwargs == {"kubeconfig_path": "/b/kube"} + assert a.request_timeout == 10 + assert b.request_timeout == 20 diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_k8s_registry.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_k8s_registry.py new file mode 100644 index 0000000000..a94919a1b5 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_k8s_registry.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +from nemo_deployments_plugin.backends.labels import ( + k8s_deployment_resource_name, + k8s_volume_resource_name, + managed_by_label_selector, +) + + +def test_k8s_resource_names_match_docker_prefixes() -> None: + dep = k8s_deployment_resource_name("foo", "bar-baz") + vol = k8s_volume_resource_name("foo", "data") + assert dep.startswith("dep-") + assert vol.startswith("dep-vol-") + assert len(dep) <= 63 + assert len(vol) <= 63 + + +def test_ambiguous_workspace_name_pairs_get_distinct_k8s_names() -> None: + a = k8s_deployment_resource_name("foo", "bar-baz") + b = k8s_deployment_resource_name("foo-bar", "baz") + assert a != b + + +def test_managed_by_label_selector() -> None: + assert managed_by_label_selector() == "managed-by=nemo-deployments" diff --git a/uv.lock b/uv.lock index 91c87960a8..442bbb9c64 100644 --- a/uv.lock +++ b/uv.lock @@ -4329,12 +4329,16 @@ dependencies = [ docker = [ { name = "docker", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] +k8s = [ + { name = "kubernetes", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, +] [package.dev-dependencies] dev = [ { name = "docker", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "fastapi", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "httpx", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, + { name = "kubernetes", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pytest", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, { name = "pytest-asyncio", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or (platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] @@ -4344,17 +4348,19 @@ requires-dist = [ { name = "docker", marker = "extra == 'docker'", specifier = ">=7.0" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", specifier = ">=0.27" }, + { name = "kubernetes", marker = "extra == 'k8s'", specifier = ">=30.1.0" }, { name = "nemo-platform", editable = "packages/nemo_platform" }, { name = "nemo-platform-plugin", editable = "packages/nemo_platform_plugin" }, { name = "pydantic", specifier = ">=2.10.6" }, ] -provides-extras = ["docker"] +provides-extras = ["docker", "k8s"] [package.metadata.requires-dev] dev = [ { name = "docker", specifier = ">=7.0" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", specifier = ">=0.27" }, + { name = "kubernetes", specifier = ">=30.1.0" }, { name = "pytest", specifier = ">=8.3.4" }, { name = "pytest-asyncio", specifier = ">=0.25.3" }, ] From 76e3c05d0f2db7399511daeeddc2bc2a50c0859c Mon Sep 17 00:00:00 2001 From: Tyler Bray Date: Wed, 1 Jul 2026 09:16:29 -0700 Subject: [PATCH 2/3] fix(deployments): address PR 533 review feedback for k8s client Centralize kubernetes imports in _kubernetes_modules() so optional-dep loading stays lazy without repeated inline imports. Add KubernetesClients.close() and wire K8sDeploymentBackend.shutdown(). Fix tautological kubeconfig fallback test and narrow managed_by_filter return type. Signed-off-by: Tyler Bray --- .../backends/k8s/backend.py | 3 +- .../backends/k8s/client.py | 59 +++++++++++++------ .../backends/labels.py | 5 +- .../tests/unit/backends/k8s/test_backend.py | 9 +++ .../tests/unit/backends/k8s/test_client.py | 25 +++++++- 5 files changed, 78 insertions(+), 23 deletions(-) diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py index ffebcd1be9..27d575040e 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py @@ -47,7 +47,8 @@ def init(self) -> None: ) def shutdown(self) -> None: - pass + if hasattr(self, "_clients"): + self._clients.close() @property def executor_config(self) -> K8sExecutorConfig: diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py index 607d160659..501c2e86db 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py @@ -4,23 +4,38 @@ """Kubernetes client bootstrap for the deployments plugin. Copied from the jobs service pattern; tagged for future extraction to a shared substrate lib. + +Imports are centralized in ``_kubernetes_modules()`` rather than hoisted to module scope so +``registry`` can load without requiring the optional ``kubernetes`` package until a k8s +executor is actually constructed. """ from __future__ import annotations import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: - from kubernetes import client + from kubernetes.client import ApiClient, AppsV1Api, BatchV1Api, CoreV1Api logger = logging.getLogger(__name__) +_kubernetes_modules_cache: tuple[Any, Any] | None = None + + +def _kubernetes_modules() -> tuple[Any, Any]: + """Return ``(kubernetes.client, kubernetes.config)``, importing on first use.""" + global _kubernetes_modules_cache + if _kubernetes_modules_cache is None: + from kubernetes import client, config + + _kubernetes_modules_cache = (client, config) + return _kubernetes_modules_cache -def build_api_client(*, kubeconfig_path: str | None = None) -> client.ApiClient: - """Create an ``ApiClient`` for the given kubeconfig (in-cluster when path is unset).""" - from kubernetes import client, config +def build_api_client(*, kubeconfig_path: str | None = None) -> ApiClient: + """Create an ``ApiClient`` for the given kubeconfig (in-cluster when path is unset).""" + client, config = _kubernetes_modules() configuration = client.Configuration() if kubeconfig_path: config.load_kube_config(config_file=kubeconfig_path, client_configuration=configuration) @@ -38,17 +53,17 @@ class KubernetesClients: def __init__(self, *, kubeconfig_path: str | None = None, request_timeout: int = 60) -> None: self._kubeconfig_path = kubeconfig_path self._request_timeout = request_timeout - self._api_client: client.ApiClient | None = None - self._core_v1: client.CoreV1Api | None = None - self._apps_v1: client.AppsV1Api | None = None - self._batch_v1: client.BatchV1Api | None = None + self._api_client: ApiClient | None = None + self._core_v1: CoreV1Api | None = None + self._apps_v1: AppsV1Api | None = None + self._batch_v1: BatchV1Api | None = None @property def request_timeout(self) -> int: """Per-request timeout (seconds) for Kubernetes API calls in later phases.""" return self._request_timeout - def _api(self) -> client.ApiClient: + def _api(self) -> ApiClient: if self._api_client is None: self._api_client = build_api_client(kubeconfig_path=self._kubeconfig_path) logger.debug( @@ -59,25 +74,31 @@ def _api(self) -> client.ApiClient: return self._api_client @property - def core_v1(self) -> client.CoreV1Api: + def core_v1(self) -> CoreV1Api: if self._core_v1 is None: - from kubernetes import client - + client, _ = _kubernetes_modules() self._core_v1 = client.CoreV1Api(self._api()) return self._core_v1 @property - def apps_v1(self) -> client.AppsV1Api: + def apps_v1(self) -> AppsV1Api: if self._apps_v1 is None: - from kubernetes import client - + client, _ = _kubernetes_modules() self._apps_v1 = client.AppsV1Api(self._api()) return self._apps_v1 @property - def batch_v1(self) -> client.BatchV1Api: + def batch_v1(self) -> BatchV1Api: if self._batch_v1 is None: - from kubernetes import client - + client, _ = _kubernetes_modules() self._batch_v1 = client.BatchV1Api(self._api()) return self._batch_v1 + + def close(self) -> None: + """Release the underlying ``ApiClient`` connection pool, if created.""" + if self._api_client is not None: + self._api_client.close() + self._api_client = None + self._core_v1 = None + self._apps_v1 = None + self._batch_v1 = None diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py index 94a36c5ac7..6ad57b90d6 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py @@ -59,6 +59,7 @@ def deployment_identity_labels( config_name: str, backoff_limit: int = 6, ) -> dict[str, str]: + """Return identity labels attached to deployment backend resources.""" return { MANAGED_BY_KEY: MANAGED_BY_LABEL, DEPLOYMENT_WORKSPACE_LABEL: workspace, @@ -70,6 +71,7 @@ def deployment_identity_labels( def volume_identity_labels(workspace: str, name: str) -> dict[str, str]: + """Return identity labels attached to volume backend resources.""" return { MANAGED_BY_KEY: MANAGED_BY_LABEL, VOLUME_WORKSPACE_LABEL: workspace, @@ -77,7 +79,8 @@ def volume_identity_labels(workspace: str, name: str) -> dict[str, str]: } -def managed_by_filter() -> dict[str, str | bool]: +def managed_by_filter() -> dict[str, str]: + """Return a Docker SDK filter dict for plugin-managed resources.""" return {"label": f"{MANAGED_BY_KEY}={MANAGED_BY_LABEL}"} diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py index f21e0ea600..2021aa64e6 100644 --- a/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py @@ -3,6 +3,8 @@ from __future__ import annotations +from unittest.mock import MagicMock + import pytest from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend @@ -22,3 +24,10 @@ async def test_create_deployment_not_implemented_yet(k8s_backend: K8sDeploymentB def test_executor_config_parsed_from_dict(k8s_backend: K8sDeploymentBackend) -> None: assert k8s_backend.executor_config.default_namespace == "nemo-deployments" assert k8s_backend.executor_config.request_timeout == 30 + + +def test_shutdown_closes_kubernetes_clients(k8s_backend: K8sDeploymentBackend) -> None: + mock_clients = MagicMock() + k8s_backend._clients = mock_clients + k8s_backend.shutdown() + mock_clients.close.assert_called_once() diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py index ef0c8db3a8..917d4deaaa 100644 --- a/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py @@ -3,6 +3,7 @@ from __future__ import annotations +from collections.abc import Iterator from unittest.mock import MagicMock, patch import pytest @@ -12,6 +13,15 @@ from nemo_platform import AsyncNeMoPlatform +@pytest.fixture(autouse=True) +def _reset_kubernetes_modules_cache() -> Iterator[None]: + import nemo_deployments_plugin.backends.k8s.client as k8s_client + + k8s_client._kubernetes_modules_cache = None + yield + k8s_client._kubernetes_modules_cache = None + + def test_k8s_backend_registered() -> None: assert "k8s" in BACKEND_CLASSES assert BACKEND_CLASSES["k8s"] is K8sDeploymentBackend @@ -70,10 +80,10 @@ def test_build_api_client_falls_back_to_kubeconfig() -> None: patch("kubernetes.config.load_incluster_config", side_effect=config.ConfigException("not in cluster")), patch("kubernetes.config.load_kube_config") as mock_kube, patch("kubernetes.client.ApiClient"), - patch("kubernetes.client.Configuration"), + patch("kubernetes.client.Configuration") as mock_configuration, ): build_api_client() - mock_kube.assert_called_once_with(client_configuration=mock_kube.call_args.kwargs["client_configuration"]) + mock_kube.assert_called_once_with(client_configuration=mock_configuration.return_value) def test_build_api_client_honors_explicit_kubeconfig_path() -> None: @@ -100,3 +110,14 @@ def test_each_kubernetes_clients_instance_uses_its_kubeconfig_path() -> None: assert mock_build.call_args_list[1].kwargs == {"kubeconfig_path": "/b/kube"} assert a.request_timeout == 10 assert b.request_timeout == 20 + + +def test_kubernetes_clients_close_releases_api_client() -> None: + mock_api_client = MagicMock() + with patch("nemo_deployments_plugin.backends.k8s.client.build_api_client", return_value=mock_api_client): + clients = KubernetesClients() + _ = clients.core_v1 + clients.close() + mock_api_client.close.assert_called_once() + assert clients._api_client is None + assert clients._core_v1 is None From 10dd091d7e2276de6c86a9290ac80107e96d9cd2 Mon Sep 17 00:00:00 2001 From: Tyler Bray Date: Wed, 1 Jul 2026 09:45:54 -0700 Subject: [PATCH 3/3] fix(deployments): address mckornfield PR 533 review comments Drop docker/labels.py re-export shim; import shared backends.labels directly. Remove per-instance kubeconfig busywork test. Validate default_namespace as a DNS-1123 label. Document label prefix/workspace-key and ApiClient close behavior. Signed-off-by: Tyler Bray --- .../backends/docker/backend.py | 18 ++++----- .../backends/docker/containers.py | 2 +- .../backends/docker/gpu.py | 2 +- .../backends/docker/labels.py | 38 ------------------- .../backends/docker/ports.py | 2 +- .../backends/docker/volumes.py | 2 +- .../backends/k8s/backend.py | 7 +++- .../backends/k8s/client.py | 6 ++- .../backends/k8s/config.py | 14 ++++++- .../backends/labels.py | 7 ++++ .../backends/docker/test_docker_backend.py | 2 +- .../integration/test_reconcile_docker.py | 2 +- .../backends/docker/test_backend_mocked.py | 2 +- .../tests/unit/backends/docker/test_gpu.py | 2 +- .../unit/backends/docker/test_idempotency.py | 2 +- .../tests/unit/backends/docker/test_labels.py | 2 +- .../tests/unit/backends/k8s/test_backend.py | 6 +++ .../tests/unit/backends/k8s/test_client.py | 13 ------- 18 files changed, 56 insertions(+), 73 deletions(-) delete mode 100644 plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py index ed91565423..d67994a70b 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/backend.py @@ -31,7 +31,15 @@ validate_config_for_docker, ) from nemo_deployments_plugin.backends.docker.gpu import GPUAllocationError, get_shared_gpu_pool -from nemo_deployments_plugin.backends.docker.labels import ( +from nemo_deployments_plugin.backends.docker.ports import find_available_port +from nemo_deployments_plugin.backends.docker.probes import check_readiness_probe, host_url_for_port +from nemo_deployments_plugin.backends.docker.status import ( + LOG_MAX_CHARS, + map_docker_state_to_starting, + map_exited_status, + missing_container_status, +) +from nemo_deployments_plugin.backends.labels import ( BACKOFF_LIMIT_LABEL, CONFIG_NAME_LABEL, DEPLOYMENT_NAME_LABEL, @@ -43,14 +51,6 @@ deployment_key, managed_by_filter, ) -from nemo_deployments_plugin.backends.docker.ports import find_available_port -from nemo_deployments_plugin.backends.docker.probes import check_readiness_probe, host_url_for_port -from nemo_deployments_plugin.backends.docker.status import ( - LOG_MAX_CHARS, - map_docker_state_to_starting, - map_exited_status, - missing_container_status, -) from nemo_deployments_plugin.constants import MANAGED_BY_LABEL from nemo_deployments_plugin.entities import Container, Deployment, DeploymentConfig from nemo_deployments_plugin.types import Endpoint, RestartPolicy diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/containers.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/containers.py index 55f30055f3..00b50cd6e4 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/containers.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/containers.py @@ -7,7 +7,7 @@ from typing import Any -from nemo_deployments_plugin.backends.docker.labels import docker_volume_name +from nemo_deployments_plugin.backends.labels import docker_volume_name from nemo_deployments_plugin.entities import Container, DeploymentConfig, DockerDeploymentConfig, VolumeMount from nemo_deployments_plugin.types import RestartPolicy diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/gpu.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/gpu.py index ab1a69bcc6..a1807600f8 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/gpu.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/gpu.py @@ -131,7 +131,7 @@ def parse_gpu_device_ids(device_requests: list[Any] | None) -> list[int]: def discover_managed_gpu_allocations(client: docker.DockerClient) -> dict[str, list[int]]: """Return workload_id -> GPU IDs for running deployment-managed containers.""" - from nemo_deployments_plugin.backends.docker.labels import ( + from nemo_deployments_plugin.backends.labels import ( DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL, MANAGED_BY_KEY, diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py deleted file mode 100644 index a2627250f2..0000000000 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/labels.py +++ /dev/null @@ -1,38 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""Docker resource naming and identity labels (re-exported from shared ``backends.labels``).""" - -from nemo_deployments_plugin.backends.labels import ( - BACKOFF_LIMIT_LABEL, - CONFIG_NAME_LABEL, - DEPLOYMENT_NAME_LABEL, - DEPLOYMENT_WORKSPACE_LABEL, - MANAGED_BY_KEY, - RESTART_POLICY_LABEL, - VOLUME_NAME_LABEL, - VOLUME_WORKSPACE_LABEL, - container_name, - deployment_identity_labels, - deployment_key, - docker_volume_name, - managed_by_filter, - volume_identity_labels, -) - -__all__ = [ - "BACKOFF_LIMIT_LABEL", - "CONFIG_NAME_LABEL", - "DEPLOYMENT_NAME_LABEL", - "DEPLOYMENT_WORKSPACE_LABEL", - "MANAGED_BY_KEY", - "RESTART_POLICY_LABEL", - "VOLUME_NAME_LABEL", - "VOLUME_WORKSPACE_LABEL", - "container_name", - "deployment_identity_labels", - "deployment_key", - "docker_volume_name", - "managed_by_filter", - "volume_identity_labels", -] diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/ports.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/ports.py index cc9239b7f8..d9516a1f4f 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/ports.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/ports.py @@ -11,7 +11,7 @@ import socket from typing import TYPE_CHECKING -from nemo_deployments_plugin.backends.docker.labels import managed_by_filter +from nemo_deployments_plugin.backends.labels import managed_by_filter import docker diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/volumes.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/volumes.py index 35047e012c..3df7a2b1fe 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/volumes.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/docker/volumes.py @@ -10,7 +10,7 @@ from typing import Any from nemo_deployments_plugin.backends.base import VolumeStatusUpdate -from nemo_deployments_plugin.backends.docker.labels import docker_volume_name, volume_identity_labels +from nemo_deployments_plugin.backends.labels import docker_volume_name, volume_identity_labels import docker diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py index 27d575040e..acafd58274 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/backend.py @@ -26,7 +26,12 @@ class K8sDeploymentBackend(DeploymentBackend): - """Manage deployments and volumes as native Kubernetes objects.""" + """Manage deployments and volumes as native Kubernetes objects. + + Lifecycle methods not yet implemented raise ``NotImplementedError`` (not ``...``) so + accidental calls fail loudly during phased rollout; ``...`` is for ``@abstractmethod`` + stubs on the ABC itself. + """ _clients: KubernetesClients diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py index 501c2e86db..0d6ac897d4 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/client.py @@ -95,7 +95,11 @@ def batch_v1(self) -> BatchV1Api: return self._batch_v1 def close(self) -> None: - """Release the underlying ``ApiClient`` connection pool, if created.""" + """Release the underlying ``ApiClient`` connection pool, if created. + + ``CoreV1Api`` / ``AppsV1Api`` / ``BatchV1Api`` share the same ``ApiClient`` instance; + closing it invalidates the cached API wrappers (reset below). + """ if self._api_client is not None: self._api_client.close() self._api_client = None diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py index 7375a1165e..61d2abb0d0 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/k8s/config.py @@ -5,7 +5,11 @@ from __future__ import annotations -from pydantic import BaseModel, Field +import re + +from pydantic import BaseModel, Field, field_validator + +_DNS_LABEL_PATTERN = re.compile(r"^[a-z0-9]([a-z0-9-]*[a-z0-9])?$") class K8sExecutorConfig(BaseModel): @@ -18,6 +22,7 @@ class K8sExecutorConfig(BaseModel): default_namespace: str = Field( default="default", min_length=1, + max_length=63, description="Namespace for resources when entity backend_config.k8s.namespace is unset.", ) request_timeout: int = Field( @@ -25,3 +30,10 @@ class K8sExecutorConfig(BaseModel): ge=1, description="Kubernetes API client timeout in seconds.", ) + + @field_validator("default_namespace") + @classmethod + def _validate_default_namespace(cls, value: str) -> str: + if not _DNS_LABEL_PATTERN.fullmatch(value): + raise ValueError("default_namespace must be a lowercase DNS-1123 label (alphanumeric, interior hyphens)") + return value diff --git a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py index 6ad57b90d6..430b0835c4 100644 --- a/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py +++ b/plugins/nemo-deployments/src/nemo_deployments_plugin/backends/labels.py @@ -5,6 +5,13 @@ Naming uses ``nemo_platform_plugin.k8s_naming`` (plugins cannot import ``nmp_common``). Identity labels drive orphan cleanup and idempotency across docker and k8s backends. + +Label domain is ``nemo.nvidia.com/*`` (deployments plugin scope). Core services such as +jobs and models use ``nmp.nvidia.com/*`` today; converging prefixes is out of scope for 757. + +Deployment and volume resources use separate workspace label keys +(``deployment-workspace`` vs ``volume-workspace``) so list/watch queries can target one +resource kind without ambiguous selectors, even though the workspace value is the same string. """ from nemo_deployments_plugin.constants import MANAGED_BY_LABEL diff --git a/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py b/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py index e0ca1154f6..58a4fa0d7a 100644 --- a/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py +++ b/plugins/nemo-deployments/tests/integration/backends/docker/test_docker_backend.py @@ -12,7 +12,7 @@ from docker_availability import skip_without_docker from integration_helpers import force_remove_container from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend -from nemo_deployments_plugin.backends.docker.labels import container_name +from nemo_deployments_plugin.backends.labels import container_name from nemo_deployments_plugin.backends.registry import BACKEND_CLASSES from nemo_deployments_plugin.constants import MANAGED_BY_LABEL from nemo_deployments_plugin.entities import ( diff --git a/plugins/nemo-deployments/tests/integration/test_reconcile_docker.py b/plugins/nemo-deployments/tests/integration/test_reconcile_docker.py index bc45edb442..74dc8a678f 100644 --- a/plugins/nemo-deployments/tests/integration/test_reconcile_docker.py +++ b/plugins/nemo-deployments/tests/integration/test_reconcile_docker.py @@ -16,7 +16,7 @@ from docker_availability import skip_without_docker from integration_helpers import force_remove_container from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend -from nemo_deployments_plugin.backends.docker.labels import container_name, docker_volume_name +from nemo_deployments_plugin.backends.labels import container_name, docker_volume_name from nemo_deployments_plugin.backends.registry import BACKEND_CLASSES, ExecutorRegistry from nemo_deployments_plugin.config import ControllerConfig from nemo_deployments_plugin.entities import ( diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py index 2fba410e60..08282f8a43 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_backend_mocked.py @@ -11,7 +11,7 @@ from backends.docker.docker_helpers import container_attrs, sample_config from docker.errors import APIError, NotFound from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend -from nemo_deployments_plugin.backends.docker.labels import ( +from nemo_deployments_plugin.backends.labels import ( CONFIG_NAME_LABEL, DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL, diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_gpu.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_gpu.py index 91fe6330a6..91a60e43d7 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_gpu.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_gpu.py @@ -14,7 +14,7 @@ get_shared_gpu_pool, parse_gpu_device_ids, ) -from nemo_deployments_plugin.backends.docker.labels import DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL +from nemo_deployments_plugin.backends.labels import DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL from nemo_deployments_plugin.constants import MANAGED_BY_LABEL diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py index 0c68cce0a4..2b1d822359 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_idempotency.py @@ -10,7 +10,7 @@ import pytest from backends.docker.docker_helpers import container_attrs, sample_config from nemo_deployments_plugin.backends.docker.backend import DockerDeploymentBackend -from nemo_deployments_plugin.backends.docker.labels import ( +from nemo_deployments_plugin.backends.labels import ( CONFIG_NAME_LABEL, DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL, diff --git a/plugins/nemo-deployments/tests/unit/backends/docker/test_labels.py b/plugins/nemo-deployments/tests/unit/backends/docker/test_labels.py index 949a579bf2..9466919130 100644 --- a/plugins/nemo-deployments/tests/unit/backends/docker/test_labels.py +++ b/plugins/nemo-deployments/tests/unit/backends/docker/test_labels.py @@ -7,7 +7,7 @@ import re -from nemo_deployments_plugin.backends.docker.labels import ( +from nemo_deployments_plugin.backends.labels import ( CONFIG_NAME_LABEL, DEPLOYMENT_NAME_LABEL, DEPLOYMENT_WORKSPACE_LABEL, diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py index 2021aa64e6..07a2af7493 100644 --- a/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_backend.py @@ -7,6 +7,7 @@ import pytest from nemo_deployments_plugin.backends.k8s.backend import K8sDeploymentBackend +from nemo_deployments_plugin.backends.k8s.config import K8sExecutorConfig @pytest.mark.asyncio @@ -31,3 +32,8 @@ def test_shutdown_closes_kubernetes_clients(k8s_backend: K8sDeploymentBackend) - k8s_backend._clients = mock_clients k8s_backend.shutdown() mock_clients.close.assert_called_once() + + +def test_default_namespace_rejects_invalid_dns_label() -> None: + with pytest.raises(ValueError, match="default_namespace must be a lowercase DNS-1123 label"): + K8sExecutorConfig(default_namespace="X") diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py index 917d4deaaa..c5935ed5c0 100644 --- a/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_client.py @@ -99,19 +99,6 @@ def test_build_api_client_honors_explicit_kubeconfig_path() -> None: mock_incluster.assert_not_called() -def test_each_kubernetes_clients_instance_uses_its_kubeconfig_path() -> None: - with patch("nemo_deployments_plugin.backends.k8s.client.build_api_client") as mock_build: - mock_build.return_value = MagicMock() - a = KubernetesClients(kubeconfig_path="/a/kube", request_timeout=10) - b = KubernetesClients(kubeconfig_path="/b/kube", request_timeout=20) - _ = a.core_v1 - _ = b.core_v1 - assert mock_build.call_args_list[0].kwargs == {"kubeconfig_path": "/a/kube"} - assert mock_build.call_args_list[1].kwargs == {"kubeconfig_path": "/b/kube"} - assert a.request_timeout == 10 - assert b.request_timeout == 20 - - def test_kubernetes_clients_close_releases_api_client() -> None: mock_api_client = MagicMock() with patch("nemo_deployments_plugin.backends.k8s.client.build_api_client", return_value=mock_api_client):