diff --git a/k8s/helm/templates/core/controller-role.yaml b/k8s/helm/templates/core/controller-role.yaml index b3a5f3604a..712eb2fb96 100644 --- a/k8s/helm/templates/core/controller-role.yaml +++ b/k8s/helm/templates/core/controller-role.yaml @@ -18,7 +18,7 @@ rules: verbs: ["get", "list"] - apiGroups: ["apps"] resources: ["deployments"] - verbs: ["get", "list", "watch"] + verbs: ["get", "list", "watch", "create", "delete"] - apiGroups: ["batch"] resources: ["jobs"] verbs: ["create", "get", "list", "watch", "update", "patch", "delete"] @@ -28,6 +28,9 @@ rules: - apiGroups: [""] resources: ["configmaps"] verbs: ["get", "list", "create", "delete"] +- apiGroups: [""] + resources: ["services"] + verbs: ["get", "list", "create", "delete"] - apiGroups: [""] resources: ["secrets"] verbs: ["create", "delete"] diff --git a/plugins/nemo-deployments/README.md b/plugins/nemo-deployments/README.md index 6b2abf8d9e..1d5e6d174d 100644 --- a/plugins/nemo-deployments/README.md +++ b/plugins/nemo-deployments/README.md @@ -56,3 +56,51 @@ from the hyphen-joined string, so pairs like ``foo``/``bar-baz`` and ``nemo_platform_plugin.k8s_naming`` (same module used by the models service). Orphan cleanup matches identity labels, not names alone; existing containers keep their old names after upgrade. + +## Kubernetes executors + +The k8s backend emits native `apps/v1.Deployment` + `v1.Service` for +`restart_policy: Always` workloads, `batch/v1.Job` for finite (`Never`/ +`OnFailure`) workloads, and `v1.PersistentVolumeClaim` for volumes — no +`k8s-nim-operator` dependency. Configure a named executor in platform YAML: + +```yaml +deployments: + executors: + - name: local-k8s + backend: k8s + config: + kubeconfig_path: /path/to/kubeconfig # unset: in-cluster config, then default kubeconfig + default_namespace: default # namespace the controller's ServiceAccount has RBAC in + request_timeout: 60 + default_executor: local-k8s +``` + +Entity-level `backend_config.k8s.namespace` overrides `default_namespace` per +deployment/volume; it must be a namespace the controller's ServiceAccount has +RBAC in (see below). + +### RBAC + +The `DeploymentsController` runs inside the `nmp-core` controller pod +(registered via the `nemo.controllers` entry point), so it reuses that pod's +existing ServiceAccount and Role rather than a dedicated one. The deploy +chart's `k8s/helm/templates/core/controller-role.yaml` grants that Role the +verbs the k8s backend needs in `.Release.Namespace`: `get`/`list`/`watch` on +pods, `get`/`list` on pods/log, `create`/`get`/`list`/`watch`/`update`/`patch`/`delete` on +`batch/v1.Job`, `get`/`list`/`create`/`delete` on PVCs, ConfigMaps, and +Services, and `get`/`list`/`watch`/`create`/`delete` on `apps/v1.Deployment`. + +This Role is namespace-scoped to the release namespace. Pointing +`backend_config.k8s.namespace` at a namespace outside the release namespace +requires additional RBAC that the chart does not provision today; +namespace-per-workspace provisioning is a documented future enhancement. + +### Native sidecars + +The LoRA-adapter-style native sidecar pattern (an `init_containers` entry with +`restart_policy: "Always"`) requires Kubernetes >= 1.29. On older clusters, +omit the per-container `restart_policy` on init containers and run that +container as a regular main container instead — the compiler does not +fall back to legacy sidecar emulation (emptyDir readiness files, etc.) +automatically. diff --git a/plugins/nemo-deployments/tests/unit/backends/k8s/test_rbac_manifest.py b/plugins/nemo-deployments/tests/unit/backends/k8s/test_rbac_manifest.py new file mode 100644 index 0000000000..c0f3add6d0 --- /dev/null +++ b/plugins/nemo-deployments/tests/unit/backends/k8s/test_rbac_manifest.py @@ -0,0 +1,77 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Assert the deploy chart's controller Role grants the k8s backend's required RBAC. + +The chart under ``k8s/helm`` declares an unconditional ``k8s-nim-operator`` chart +dependency, so ``helm template`` cannot render without that subchart present in +``charts/`` (fetched from an NGC repo) — a network dependency this suite must not +require. Assertions instead parse the static YAML rule entries directly out of the +Go-template source, which is safe because the base RBAC rules (unlike the +Volcano/NIM-Operator blocks) are plain YAML with no Helm expressions inside them. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +_CONTROLLER_ROLE_RELATIVE_PATH = Path("k8s", "helm", "templates", "core", "controller-role.yaml") + + +def _repo_root() -> Path: + for parent in Path(__file__).resolve().parents: + if (parent / _CONTROLLER_ROLE_RELATIVE_PATH).is_file(): + return parent + raise FileNotFoundError(f"Could not locate repo root containing {_CONTROLLER_ROLE_RELATIVE_PATH}") + + +def _controller_role_source() -> str: + role_path = _repo_root() / _CONTROLLER_ROLE_RELATIVE_PATH + content = role_path.read_text() + # Only the Role definition (before the "---" separator) is relevant; the + # RoleBinding that follows has no `rules:` block to collide with. + role_section, _, _ = content.partition("\n---\n") + return role_section + + +def _rule_verbs(content: str, *, api_group: str, resource: str) -> list[str]: + """Extract the verbs list for a ``- apiGroups: [...]\\n resources: [...]`` rule.""" + pattern = re.compile( + r'apiGroups:\s*\[\s*"{}"\s*\]\s*\n\s*resources:\s*\[\s*"{}"\s*\]\s*\n\s*verbs:\s*\[(.*?)\]'.format( + re.escape(api_group), re.escape(resource) + ) + ) + match = pattern.search(content) + assert match is not None, f"No rule found for apiGroups={api_group!r} resources={resource!r}" + return [verb.strip().strip('"') for verb in match.group(1).split(",")] + + +def test_deployments_rule_grants_create_and_delete() -> None: + """The k8s backend creates and deletes apps/v1.Deployment objects.""" + verbs = _rule_verbs(_controller_role_source(), api_group="apps", resource="deployments") + assert {"get", "list", "watch", "create", "delete"} <= set(verbs) + + +def test_services_rule_present() -> None: + """The k8s backend creates a v1.Service alongside restart_policy=Always Deployments.""" + verbs = _rule_verbs(_controller_role_source(), api_group="", resource="services") + assert {"get", "list", "create", "delete"} <= set(verbs) + + +@pytest.mark.parametrize( + ("api_group", "resource", "required_verbs"), + [ + ("", "pods", {"get", "list", "watch"}), + ("", "pods/log", {"get", "list"}), + ("batch", "jobs", {"create", "get", "list", "watch", "update", "patch", "delete"}), + ("", "persistentvolumeclaims", {"get", "list", "create", "delete"}), + ("", "configmaps", {"get", "list", "create", "delete"}), + ], +) +def test_base_rules_unaffected(api_group: str, resource: str, required_verbs: set[str]) -> None: + """Regression guard: pre-existing k8s backend RBAC rules are untouched.""" + verbs = _rule_verbs(_controller_role_source(), api_group=api_group, resource=resource) + assert required_verbs <= set(verbs)