diff --git a/docs/set-up/config-reference.mdx b/docs/set-up/config-reference.mdx index bfbb07510c..434afdbf4f 100644 --- a/docs/set-up/config-reference.mdx +++ b/docs/set-up/config-reference.mdx @@ -529,12 +529,16 @@ models: default_user_id: # Default group ID for NIM containers (security context) default_group_id: + # Default user ID for vLLM puller + server pods (security context). Defaults to 2000 to match the upstream vLLM image's 'vllm' user, which has an /etc/passwd entry (avoids torch getpwuid crashes from an unknown uid). | default: 2000 + default_vllm_user_id: 2000 + # Default group ID / fsGroup for vLLM puller + server pods. Defaults to 0 (root group) to match the upstream vLLM image and keep weights readable across the puller and server pods. | default: 0 + default_vllm_group_id: 0 # Kubernetes secret name for Files service authentication (HF_TOKEN) | default: 'nemo-models-files-token' files_auth_secret: nemo-models-files-token # The name of the image pull secret for the modelPuller image | default: 'nvcrimagepullsecret' huggingface_model_puller_image_pull_secret: nvcrimagepullsecret - # BusyBox image repository used by plugin init containers. | default: 'busybox' - busybox_image: busybox + # BusyBox image repository used by plugin init containers. Fully qualified (docker.io/library/...) so it resolves on container runtimes that enforce fully-qualified image names (short names like 'busybox' fail there). | default: 'docker.io/library/busybox' + busybox_image: docker.io/library/busybox # BusyBox image tag used by plugin init containers. | default: 'latest' busybox_image_tag: latest # NGC API key secret name for pulling NIM images | default: 'ngc-api' @@ -543,10 +547,18 @@ models: default_nimservice_image: nvcr.io/nim/nvidia/llm-nim # Default NIMService image tag (used if not specified in deployment config) | default: '1.13.1' default_nimservice_image_tag: 1.13.1 + # Default vLLM server image repository (used if not specified in deployment config) | default: 'vllm/vllm-openai' + default_vllm_image: vllm/vllm-openai + # Default vLLM server image tag (used if not specified in deployment config) | default: 'v0.22.1' + default_vllm_image_tag: v0.22.1 # Default guided decoding backend for NIM (e.g., 'outlines', 'auto', 'lm-format-enforcer') | default: 'outlines' nim_guided_decoding_backend: outlines # Kubernetes namespace for NIM deployments (defaults to controller's namespace if not set) namespace: + # ServiceAccount name for directly-emitted vLLM Deployment pods and the weight-puller Job. If not set, the namespace default ServiceAccount is used. + service_account_name: + # Shared memory (/dev/shm) size limit for vLLM Deployment pods (e.g. '8Gi'). If not set, the emptyDir uses the node default size. + default_shared_memory_size_limit: # Default Kubernetes resource requirements for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: {'requests': {'cpu': '2', 'memory': '8Gi'}, 'limits': {'memory': '16Gi'}} default_resources: # Default Kubernetes tolerations for all NIM deployments. Can be overridden per-deployment via k8s_nim_operator_config. Example: [{'key': 'nvidia.com/gpu', 'operator': 'Exists', 'effect': 'NoSchedule'}] diff --git a/services/core/inference-gateway/tests/integration/conftest.py b/services/core/inference-gateway/tests/integration/conftest.py index cf79a13bc4..6b3ec18dd4 100644 --- a/services/core/inference-gateway/tests/integration/conftest.py +++ b/services/core/inference-gateway/tests/integration/conftest.py @@ -87,29 +87,19 @@ def init(self) -> None: """No-op init for mock backend.""" pass - async def create_model_deployment( - self, - deployment: Any, - config: Any, - model_entity: Any = None, - ) -> DeploymentStatusUpdate: + async def create_model_deployment(self, ctx: Any) -> DeploymentStatusUpdate: """Record call and return configured response.""" - self.create_calls.append((deployment, config, model_entity)) + self.create_calls.append((ctx.model_deployment, ctx.model_deployment_config, ctx.model_entity)) return self.create_response - async def update_model_deployment( - self, - deployment: Any, - config: Any, - model_entity: Any = None, - ) -> DeploymentStatusUpdate: + async def update_model_deployment(self, ctx: Any) -> DeploymentStatusUpdate: """Record call and return configured response.""" - self.update_calls.append((deployment, config, model_entity)) + self.update_calls.append((ctx.model_deployment, ctx.model_deployment_config, ctx.model_entity)) return self.create_response - async def get_model_deployment_status(self, deployment: Any) -> DeploymentStatusUpdate: + async def get_model_deployment_status(self, ctx: Any) -> DeploymentStatusUpdate: """Record call and return configured response.""" - self.status_calls.append(deployment) + self.status_calls.append(ctx.model_deployment) return self.status_response async def delete_model_deployment(self, deployment: Any) -> DeploymentStatusUpdate: diff --git a/services/core/models/src/nmp/core/models/controllers/backends/backends.py b/services/core/models/src/nmp/core/models/controllers/backends/backends.py index 65a83f5e5e..23662a8bef 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/backends.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/backends.py @@ -4,13 +4,11 @@ """Base backend interface for Models Controller service.""" from abc import ABC, abstractmethod -from typing import Any, Dict, Optional +from typing import Any, Dict from nemo_platform import AsyncNeMoPlatform from nemo_platform.types.inference import ModelDeploymentStatus -from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig -from nemo_platform.types.models.model_entity import ModelEntity +from nmp.core.models.controllers.context import ModelContext from pydantic import BaseModel @@ -67,15 +65,12 @@ def shutdown(self) -> None: ... @abstractmethod - async def create_model_deployment( - self, deployment: ModelDeployment, config: ModelDeploymentConfig, model_entity: Optional[ModelEntity] = None - ) -> DeploymentStatusUpdate: + async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Create a new model deployment. Args: - deployment: The ModelDeployment object to create - config: The ModelDeploymentConfig for this deployment - model_entity: Optional Model entity from Entity Store (contains peft, artifact, etc.) + ctx: The reconciliation context bundling the ModelDeployment, its + ModelDeploymentConfig, and the optional Model entity. Returns: DeploymentStatusUpdate with the current status after creation attempt @@ -86,15 +81,13 @@ async def create_model_deployment( ... @abstractmethod - async def update_model_deployment( - self, deployment: ModelDeployment, config: ModelDeploymentConfig, model_entity: Optional[ModelEntity] = None - ) -> DeploymentStatusUpdate: + async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Update an existing model deployment. Args: - deployment: The ModelDeployment object with updated configuration - config: The ModelDeploymentConfig for this deployment (may be a new version) - model_entity: Optional Model entity from Entity Store (contains peft, artifact, etc.) + ctx: The reconciliation context bundling the ModelDeployment, its + (possibly new-version) ModelDeploymentConfig, and the optional + Model entity. Returns: DeploymentStatusUpdate with the current status after update attempt @@ -105,11 +98,14 @@ async def update_model_deployment( ... @abstractmethod - async def get_model_deployment_status(self, deployment: ModelDeployment) -> DeploymentStatusUpdate: + async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Get the current status of a model deployment. Args: - deployment: The ModelDeployment object to check + ctx: The reconciliation context bundling the ModelDeployment, its + ModelDeploymentConfig, and the optional Model entity. Some backends + need the config to advance creation (e.g. the k8s vLLM path emits + the serving Deployment once the weight-puller Job completes). Returns: DeploymentStatusUpdate with the current deployment status diff --git a/services/core/models/src/nmp/core/models/controllers/backends/docker/backend.py b/services/core/models/src/nmp/core/models/controllers/backends/docker/backend.py index ce9d824071..7bee18711f 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/docker/backend.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/docker/backend.py @@ -12,14 +12,12 @@ import asyncio import os from logging import getLogger -from typing import Any, Optional +from typing import Any import httpx from docker.errors import APIError, NotFound from nemo_platform import NotFoundError from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig -from nemo_platform.types.models.model_entity import ModelEntity from nmp.common.config import get_platform_config from nmp.common.docker.gpu_pool import DockerGPUPool from nmp.common.resources import SharedResourceManager @@ -37,6 +35,7 @@ NGC_IMAGE_REGISTRY_USER_NAME, DockerDeploymentCreationReconciler, ) +from nmp.core.models.controllers.context import ModelContext from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import ReadTimeout from urllib3.exceptions import ReadTimeoutError as Urllib3ReadTimeoutError @@ -184,17 +183,15 @@ async def _ensure_ngc_login(self, ngc_api_key: str | None) -> None: # ServiceBackend CRUD interface # ================================================================== - async def create_model_deployment( - self, - deployment: ModelDeployment, - config: ModelDeploymentConfig, - model_entity: Optional[ModelEntity] = None, - ) -> DeploymentStatusUpdate: + async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Create a new model deployment as a Docker container. Resolves NGC credentials and delegates the multi-stage creation pipeline to :class:`DockerDeploymentCreationReconciler`. """ + deployment = ctx.model_deployment + config = ctx.model_deployment_config + model_entity = ctx.model_entity resolved_ngc_key = await self._resolve_ngc_api_key() await self._ensure_ngc_login(resolved_ngc_key) @@ -205,25 +202,22 @@ async def create_model_deployment( resolved_ngc_key, ) - async def update_model_deployment( - self, - deployment: ModelDeployment, - config: ModelDeploymentConfig, - model_entity: Optional[ModelEntity] = None, - ) -> DeploymentStatusUpdate: + async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Update a model deployment by recreating the container.""" + deployment = ctx.model_deployment logger.info(f"Updating Docker deployment: {deployment.workspace}/{deployment.name}") delete_result = await self.delete_model_deployment(deployment.workspace, deployment.name) if delete_result.status == "ERROR": return delete_result - return await self.create_model_deployment(deployment, config, model_entity) + return await self.create_model_deployment(ctx) - async def get_model_deployment_status(self, deployment: ModelDeployment) -> DeploymentStatusUpdate: + async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Get the status of a Docker model deployment. While the deployment is still progressing through the creation pipeline this delegates to the reconciler's ``advance`` method. """ + deployment = ctx.model_deployment if self._reconciler.is_deploying(deployment.workspace, deployment.name): deployment_key = self._reconciler.get_deployment_key(deployment.workspace, deployment.name) return await self._reconciler.advance(deployment_key) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/docker/creation_reconciler.py b/services/core/models/src/nmp/core/models/controllers/backends/docker/creation_reconciler.py index f2922ca7b1..4ddac4d850 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/docker/creation_reconciler.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/docker/creation_reconciler.py @@ -35,14 +35,27 @@ from nmp.core.models.app import ModelWeightsType, get_model_weights_type, is_multi_llm_image, parse_model_name_revision from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER from nmp.core.models.app.utils import _get_k8s_safe_name +from nmp.core.models.controllers.backends import vllm_compiler from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate -from nmp.core.models.controllers.backends.common import DeploymentConfigView, deployment_config_view -from nmp.core.models.controllers.backends.docker import vllm_compiler +from nmp.core.models.controllers.backends.common import deployment_config_view from nmp.core.models.controllers.backends.docker.config import ( MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE, MODELS_DOCKER_NIM_MULTI_GPU_SHM_SIZE_PER_GPU, DockerBackendConfig, ) +from nmp.core.models.controllers.backends.engine import ( + ENGINE_HEALTH_PATHS, + ENGINE_LABEL, + ENGINE_NIM, + ENGINE_VLLM, + HEALTH_PATH_LABEL, +) +from nmp.core.models.controllers.backends.engine import ( + config_engine as _config_engine, +) +from nmp.core.models.controllers.backends.engine import ( + resolve_health_path as _resolve_health_path, +) from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import ReadTimeout from tenacity import before_sleep_log, retry, stop_after_attempt, wait_exponential @@ -65,44 +78,6 @@ NGC_IMAGE_REGISTRY = os.getenv("NGC_IMAGE_REGISTRY", "nvcr.io") NGC_IMAGE_REGISTRY_USER_NAME = os.getenv("NGC_IMAGE_REGISTRY_USER_NAME", "$oauthtoken") -ENGINE_NIM = "nim" -ENGINE_VLLM = "vllm" -ENGINE_GENERIC = "generic" - -# Docker label recording the engine, read back at status time to pick the health probe. -ENGINE_LABEL = "nmp.nvidia.com/engine" - -# Docker label recording the resolved readiness-probe path, read back at status -# time. Stamped at create so status polling doesn't need the deployment config. -HEALTH_PATH_LABEL = "nmp.nvidia.com/health-path" - -# Per-engine readiness probe paths (relative to the container host URL). -ENGINE_HEALTH_PATHS: dict[str, str] = { - ENGINE_NIM: "/v1/health/ready", - ENGINE_VLLM: "/health", -} - - -def _config_engine(config: Any) -> str: - """Return the engine discriminant as a lowercase string (defaults to nim).""" - engine = getattr(config, "engine", None) - if engine is None: - return ENGINE_NIM - # engine may be an enum or a plain string depending on the SDK model. - return str(getattr(engine, "value", engine)).lower() - - -def _resolve_health_path(engine: str, view: DeploymentConfigView) -> str: - """Resolve the readiness-probe path for a deployment. - - Precedence: an explicit ``executor_config.health_check_path`` wins; otherwise - fall back to the engine's standard endpoint. ``generic`` containers have no - engine default, so they fall back to the NIM path unless they set their own. - """ - if getattr(view, "health_check_path", None): - return view.health_check_path - return ENGINE_HEALTH_PATHS.get(engine, ENGINE_HEALTH_PATHS[ENGINE_NIM]) - def _should_retry_docker_error(exception: BaseException) -> bool: """Determine if a Docker exception should be retried.""" diff --git a/services/core/models/src/nmp/core/models/controllers/backends/engine.py b/services/core/models/src/nmp/core/models/controllers/backends/engine.py new file mode 100644 index 0000000000..8283422f94 --- /dev/null +++ b/services/core/models/src/nmp/core/models/controllers/backends/engine.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Backend-agnostic engine dispatch + readiness-probe helpers. + +The ``engine`` discriminant on a ``ModelDeploymentConfig`` selects the compiler +path (nim / vllm / generic). These constants and helpers are shared by every +service backend (docker container labels, k8s object labels) so engine selection +and readiness-probe resolution behave identically regardless of where the +deployment runs. +""" + +from typing import Any + +from nmp.core.models.controllers.backends.common import DeploymentConfigView + +ENGINE_NIM = "nim" +ENGINE_VLLM = "vllm" +ENGINE_GENERIC = "generic" + +# Label recording the engine, read back at status time to pick the health probe. +# Used as a docker container label and a k8s object/pod label. +ENGINE_LABEL = "nmp.nvidia.com/engine" + +# Label recording the resolved readiness-probe path, read back at status time. +# Stamped at create so status polling doesn't need the deployment config. +HEALTH_PATH_LABEL = "nmp.nvidia.com/health-path" + +# Per-engine readiness probe paths (relative to the container/pod host URL). +ENGINE_HEALTH_PATHS: dict[str, str] = { + ENGINE_NIM: "/v1/health/ready", + ENGINE_VLLM: "/health", +} + + +def config_engine(config: Any) -> str: + """Return the engine discriminant as a lowercase string (defaults to nim).""" + engine = getattr(config, "engine", None) + if engine is None: + return ENGINE_NIM + # engine may be an enum or a plain string depending on the SDK model. + return str(getattr(engine, "value", engine)).lower() + + +def resolve_health_path(engine: str, view: DeploymentConfigView) -> str: + """Resolve the readiness-probe path for a deployment. + + Precedence: an explicit ``executor_config.health_check_path`` wins; otherwise + fall back to the engine's standard endpoint. ``generic`` containers have no + engine default, so they fall back to the NIM path unless they set their own. + """ + explicit_path = getattr(view, "health_check_path", None) + if explicit_path: + return explicit_path + return ENGINE_HEALTH_PATHS.get(engine, ENGINE_HEALTH_PATHS[ENGINE_NIM]) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/backend.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/backend.py index 88ba1a7737..19197b89ce 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/backend.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/backend.py @@ -1,62 +1,61 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Kubernetes NIM Operator backend implementation for Models Controller service.""" +"""Kubernetes NIM Operator backend implementation for Models Controller service. + +This ``ServiceBackend`` owns the ``nemo_platform`` SDK, determines the state of +the *API object* (ModelDeployment / ModelDeploymentConfig), resolves every input +a reconciler needs (weight source, resource names, Files endpoint) into a +:class:`ResolvedDeployment`, selects the correct reconciler by engine, and +delegates. It holds NO Kubernetes-reconciliation logic itself -- that lives in +the two reconcilers under :mod:`.reconcilers`: + +* :class:`NimOperatorReconciler` -- emits ``NIMService`` / ``NIMCache`` CRs. +* :class:`K8sReconciler` -- emits native Kubernetes objects directly (vLLM). +""" import os from logging import getLogger -from typing import Any, Dict, Optional +from typing import Optional +from urllib.parse import urljoin from kubernetes import client as k8s_client from kubernetes import config as k8s_config from kubernetes.dynamic import DynamicClient -from kubernetes.dynamic import exceptions as k8s_dynamic_exceptions from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig from nemo_platform.types.models.model_entity import ModelEntity +from nmp.common.config import get_platform_config from nmp.core.models.app import ( - ModelWeightsType, get_deployment_resource_name, get_model_weights_type, get_nimcache_resource_name, parse_model_name_revision, ) -from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend from nmp.core.models.controllers.backends.common import ( - LOG_MAX_CHARS, - LOG_TAIL_LINES, + DeploymentConfigView, deployment_config_view, deployment_elapsed_seconds, - format_duration, ) +from nmp.core.models.controllers.backends.engine import ENGINE_GENERIC, ENGINE_VLLM, config_engine from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig -from nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler import ( - compile_nimcache, - compile_nimservice, +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.base import Reconciler, ResolvedDeployment +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.k8s import K8sReconciler +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator import ( + NimOperatorReconciler, ) +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.resource_deleter import ResourceDeleter +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector import StatusProjector +from nmp.core.models.controllers.context import ModelContext logger = getLogger(__name__) -NIM_OPERATOR_GROUP = "apps.nvidia.com" -NIMSERVICE_VERSION = "v1alpha1" -NIMSERVICE_API_VERSION = f"{NIM_OPERATOR_GROUP}/{NIMSERVICE_VERSION}" -NIMSERVICE_PLURAL = "nimservices" - -NIMCACHE_VERSION = "v1alpha1" -NIMCACHE_API_VERSION = f"{NIM_OPERATOR_GROUP}/{NIMCACHE_VERSION}" -NIMCACHE_PLURAL = "nimcaches" - -POD_EVENT_TO_MESSAGE_MAP = { - "startup probe failed": "Waiting for pod to finish startup", -} - class K8sNimOperatorServiceBackend(ServiceBackend): - """Kubernetes NIM Operator backend for managing model deployments. + """Kubernetes backend for managing model deployments. - Manages ModelDeployment lifecycle by creating and managing NIMService - custom resources via the NIM Operator in Kubernetes. + Resolves API-object state and delegates reconciliation to the engine-specific + reconciler (NIM operator CRs vs. native Kubernetes objects). """ def __init__(self, nmp_sdk, config, huggingface_model_puller: str): @@ -65,10 +64,14 @@ def __init__(self, nmp_sdk, config, huggingface_model_puller: str): self._k8s_namespace: str | None = None self._backend_config: K8sNimOperatorConfig | None = None self._huggingface_model_puller = huggingface_model_puller + self._status_projector: StatusProjector | None = None + self._resource_deleter: ResourceDeleter | None = None + self._nim_reconciler: NimOperatorReconciler | None = None + self._k8s_reconciler: K8sReconciler | None = None super().__init__(nmp_sdk, config) def init(self) -> None: - """Initialize Kubernetes NIM Operator backend.""" + """Initialize Kubernetes backend and build the engine reconcilers.""" logger.info("Initializing Kubernetes NIM Operator service backend") self._backend_config = K8sNimOperatorConfig(**self._config) @@ -89,7 +92,31 @@ def init(self) -> None: self._k8s_namespace = self._get_current_namespace() logger.info(f"Models controller will deploy models to namespace: {self._k8s_namespace}") - self._validate_nim_operator_crds() + # Shared collaborators composed into both reconcilers (and used directly + # by the PENDING-timeout policy below). + self._status_projector = StatusProjector( + k8s_client_=self._k8s_client, + backend_config=self._backend_config, + k8s_namespace=self._k8s_namespace, + ) + self._resource_deleter = ResourceDeleter(k8s_namespace=self._k8s_namespace) + + self._nim_reconciler = NimOperatorReconciler( + dynamic_client=self._dynamic_client, + backend_config=self._backend_config, + k8s_namespace=self._k8s_namespace, + huggingface_model_puller=self._huggingface_model_puller, + status=self._status_projector, + deleter=self._resource_deleter, + ) + self._k8s_reconciler = K8sReconciler( + k8s_client_=self._k8s_client, + backend_config=self._backend_config, + k8s_namespace=self._k8s_namespace, + huggingface_model_puller=self._huggingface_model_puller, + status=self._status_projector, + deleter=self._resource_deleter, + ) def shutdown(self) -> None: """Shutdown Kubernetes backend and release resources.""" @@ -115,47 +142,9 @@ def _get_current_namespace(self) -> str: logger.warning("Could not determine k8s namespace, using 'default'") return "default" - def _validate_nim_operator_crds(self) -> None: - """ - Validate that NIM Operator APIs are available via API discovery. - - Raises: - RuntimeError: If required APIs are not found. This will prevent the backend - from initializing and cause the controller to fail fast. - """ - # Validate NIMService API is available - try: - self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - logger.info(f"Validated NIMService API is available: {NIMSERVICE_API_VERSION} NIMService") - except k8s_dynamic_exceptions.ResourceNotFoundError as e: - logger.error(f"NIMService CRD not found: {e}") - raise RuntimeError( - f"NIMService API ({NIMSERVICE_API_VERSION}) not found. " - f"The k8s-nim-operator must be installed before starting this backend." - ) from e - except Exception as e: - logger.exception("Unexpected error validating NIMService API") - raise RuntimeError(f"Failed to validate NIMService API ({NIMSERVICE_API_VERSION}): {e}") from e - - # Validate NIMCache API is available - try: - self._dynamic_client.resources.get( - api_version=NIMCACHE_API_VERSION, - kind="NIMCache", - ) - logger.info(f"Validated NIMCache API is available: {NIMCACHE_API_VERSION} NIMCache") - except k8s_dynamic_exceptions.ResourceNotFoundError as e: - logger.error(f"NIMCache CRD not found: {e}") - raise RuntimeError( - f"NIMCache API ({NIMCACHE_API_VERSION}) not found. " - f"The k8s-nim-operator must be installed before starting this backend." - ) from e - except Exception as e: - logger.exception("Unexpected error validating NIMCache API") - raise RuntimeError(f"Failed to validate NIMCache API ({NIMCACHE_API_VERSION}): {e}") from e + # ------------------------------------------------------------------ + # Name + weight-source resolution (API-object work owned by the backend) + # ------------------------------------------------------------------ def _get_resource_name(self, deployment: ModelDeployment) -> str: """Generate the k8s resource name for NIMService/PVC resources (63-char limit).""" @@ -171,330 +160,10 @@ def _get_nimcache_resource_name(self, deployment: ModelDeployment) -> str: """ return get_nimcache_resource_name(deployment.workspace, deployment.name) - def _get_host_url(self, resource_name: str) -> str: - """Generate the Kubernetes service host URL for a deployment.""" - return f"http://{resource_name}.{self._k8s_namespace}.svc.cluster.local:8000" - - # ------------------------------------------------------------------ - # Pod log fetching and pod lookup (best-effort diagnostics) - # ------------------------------------------------------------------ - - def _fetch_pod_logs(self, pod_name: str) -> str: - """Fetch recent pod logs for error reporting, truncated to LOG_MAX_CHARS.""" - try: - core_v1 = k8s_client.CoreV1Api(self._k8s_client) - logs = core_v1.read_namespaced_pod_log( - name=pod_name, - namespace=self._k8s_namespace, - tail_lines=LOG_TAIL_LINES, - ) - if len(logs) > LOG_MAX_CHARS: - logs = logs[-LOG_MAX_CHARS:] - return logs - except Exception as e: - logger.warning( - "Failed to retrieve pod logs for error report", extra={"pod_name": pod_name, "error": str(e)} - ) - return "" - - def _find_pod_name(self, resource_name: str) -> str | None: - """Find the most recent pod name for a k8s Deployment (best-effort).""" - try: - apps_v1 = k8s_client.AppsV1Api(self._k8s_client) - core_v1 = k8s_client.CoreV1Api(self._k8s_client) - - try: - deployment = apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) - except k8s_client.exceptions.ApiException: - return None - - if not deployment.spec.selector or not deployment.spec.selector.match_labels: - return None - - label_selector = ",".join([f"{k}={v}" for k, v in deployment.spec.selector.match_labels.items()]) - pods = core_v1.list_namespaced_pod(namespace=self._k8s_namespace, label_selector=label_selector) - - if not pods.items: - return None - - pod = max(pods.items, key=lambda p: p.metadata.creation_timestamp) - return pod.metadata.name - except Exception: - return None - - # ------------------------------------------------------------------ - # Crash loop and pending timeout error builders - # ------------------------------------------------------------------ - - def _build_pending_timeout_error( - self, - resource_name: str, - elapsed: float, - pod_name: str | None, - ) -> DeploymentStatusUpdate: - """Build ERROR status update for a PENDING timeout.""" - error_stack = self._fetch_pod_logs(pod_name) if pod_name else "" - kubectl_target = pod_name if pod_name else f"deployment/{resource_name}" - status_msg = ( - f"Deployment timed out after {format_duration(elapsed)} waiting for NIM " - f"to pass health checks (timeout: {format_duration(self._backend_config.pending_timeout_seconds)}).\n\n" - f"Inspect the NIM pod logs with:\n" - f" kubectl logs -n {self._k8s_namespace} {kubectl_target}" - ) - error_details: Dict[str, Any] = { - "reason": "pending_timeout", - "elapsed_seconds": int(elapsed), - "timeout_seconds": self._backend_config.pending_timeout_seconds, - "resource_name": resource_name, - "namespace": self._k8s_namespace, - "error_stack": error_stack if error_stack else None, - } - if pod_name: - error_details["pod_name"] = pod_name - return DeploymentStatusUpdate( - status="ERROR", - status_message=status_msg, - error_details=error_details, - host_url=None, - ) - - def _build_crash_loop_error( - self, - resource_name: str, - pod_name: str, - restart_count: int, - ) -> DeploymentStatusUpdate: - """Build ERROR status update for a crash loop.""" - error_stack = self._fetch_pod_logs(pod_name) - status_msg = ( - f"Deployment entered crash loop after {restart_count} container restarts " - f"(max: {self._backend_config.max_restart_count}).\n\n" - f"Inspect the NIM pod logs with:\n" - f" kubectl logs -n {self._k8s_namespace} {pod_name}" - ) - return DeploymentStatusUpdate( - status="ERROR", - status_message=status_msg, - error_details={ - "reason": "crash_loop", - "restart_count": restart_count, - "max_restart_count": self._backend_config.max_restart_count, - "pod_name": pod_name, - "namespace": self._k8s_namespace, - "resource_name": resource_name, - "error_stack": error_stack if error_stack else None, - }, - host_url=None, - ) - - # ------------------------------------------------------------------ - # Pod status helpers - # ------------------------------------------------------------------ - - @staticmethod - def _get_pod_restart_count(pod: k8s_client.V1Pod) -> int: - """Get the maximum restart count across all containers in a pod.""" - if not pod.status.container_statuses: - return 0 - return max((cs.restart_count or 0) for cs in pod.status.container_statuses) - - @staticmethod - def _with_restart_info(status_msg: str, restart_count: int) -> str: - """Append restart count to a status message when restarts > 0.""" - if restart_count > 0: - return f"{status_msg}, restarts: {restart_count}" - return status_msg - - def _check_crash_loop(self, pod: k8s_client.V1Pod, resource_name: str) -> DeploymentStatusUpdate | None: - """Check if a pod is in a crash loop (restart count >= max_restart_count and waiting). - - Returns a DeploymentStatusUpdate with ERROR if crash loop detected, else None. - """ - pod_name = pod.metadata.name - logger.debug("Checking pod for crash loop", extra={"pod": pod_name, "phase": pod.status.phase}) - - if not pod.status.container_statuses: - logger.debug("Pod has no container statuses", extra={"pod": pod_name}) - return None - - max_restarts = self._backend_config.max_restart_count - - for idx, container_status in enumerate(pod.status.container_statuses): - restart_count = container_status.restart_count or 0 - logger.debug( - "Container status check", - extra={"pod": pod_name, "container_index": idx, "restart_count": restart_count}, - ) - - if restart_count >= max_restarts: - if container_status.state and container_status.state.waiting: - waiting_reason = container_status.state.waiting.reason - logger.warning( - "Pod entered crash loop", - extra={ - "pod": pod_name, - "restart_count": restart_count, - "max_restarts": max_restarts, - "waiting_reason": waiting_reason, - }, - ) - return self._build_crash_loop_error(resource_name, pod_name, restart_count) - else: - logger.debug( - "Pod has restarts above threshold but is not in waiting state", - extra={"pod": pod_name, "container_index": idx, "restart_count": restart_count}, - ) - - logger.debug("Crash loop check complete, no crash loop detected", extra={"pod": pod_name}) - return None - - def _get_nimservice_status(self, resource_name: str) -> DeploymentStatusUpdate: - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - - try: - nimservice = nimservice_api.get(name=resource_name, namespace=self._k8s_namespace) - except k8s_dynamic_exceptions.NotFoundError: - logger.warning( - f"NIMService {resource_name} not found in cluster for deployment {resource_name}. " - f"The resource may have been manually deleted or removed during namespace cleanup." - ) - return DeploymentStatusUpdate( - status="LOST", - status_message="NIMService not found in cluster. Resource may have been deleted externally.", - host_url=None, - ) - - nim_status = nimservice.get("status", {}) - state = nim_status.get("state", "").lower() - - match state: - case "ready": - return DeploymentStatusUpdate( - status="READY", - status_message="", - host_url=self._get_host_url(resource_name), - ) - case "notready": - conditions = nim_status.get("conditions", []) - logger.info(f"NIMService {resource_name} is NotReady. Conditions: {conditions}") - - pod_status_result = self._get_pod_status_from_deployment(resource_name) - - return pod_status_result - case "failed": - conditions = nim_status.get("conditions", []) - logger.error(f"NIMService {resource_name} has failed. Conditions: {conditions}") - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"NIMService failed: {conditions}", - host_url=None, - ) - case _: - return DeploymentStatusUpdate( - status="PENDING", - status_message=f"NIMService in {state or 'unknown'} state", - host_url=None, - ) - - def _get_pod_status_from_deployment(self, resource_name: str) -> DeploymentStatusUpdate: - """Get status message from pod events for a deployment. - - Returns: - DeploymentStatusUpdate with status (PENDING or ERROR) and descriptive message. - Crash loop detection is performed here; PENDING timeout is handled by the caller. - """ - logger.info(f"Getting pod status for deployment: {resource_name}") - try: - apps_v1 = k8s_client.AppsV1Api(self._k8s_client) - core_v1 = k8s_client.CoreV1Api(self._k8s_client) - - try: - deployment = apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) - except k8s_client.exceptions.ApiException as e: - if e.status == 404: - return DeploymentStatusUpdate( - status="PENDING", status_message="Waiting for k8s deployment to be created", host_url=None - ) - raise - - if not deployment.spec.selector or not deployment.spec.selector.match_labels: - return DeploymentStatusUpdate( - status="PENDING", - status_message="Waiting for k8s deployment - invalid selector configuration", - host_url=None, - ) - - label_selector = ",".join([f"{k}={v}" for k, v in deployment.spec.selector.match_labels.items()]) - pods = core_v1.list_namespaced_pod(namespace=self._k8s_namespace, label_selector=label_selector) - - if not pods.items: - logger.info(f"No pods found for deployment {resource_name}") - return DeploymentStatusUpdate( - status="PENDING", status_message="Waiting for k8s deployment - no pods created yet", host_url=None - ) - - logger.info(f"Found {len(pods.items)} pod(s) for deployment {resource_name}") - - pod: k8s_client.V1Pod = max(pods.items, key=lambda p: p.metadata.creation_timestamp) - logger.info(f"Checking most recent pod: {pod.metadata.name}") - - crash_result = self._check_crash_loop(pod, resource_name) - if crash_result: - return crash_result - - restart_count = self._get_pod_restart_count(pod) - - events = core_v1.list_namespaced_event( - namespace=self._k8s_namespace, field_selector=f"involvedObject.name={pod.metadata.name}" - ) - - if not events.items: - if pod.status.phase == "Pending" and pod.status.container_statuses: - for container_status in pod.status.container_statuses: - if container_status.state and container_status.state.waiting: - reason = container_status.state.waiting.reason - message = container_status.state.waiting.message or "" - status_msg = f"{reason}: {message}" if message else reason - status_msg = self._with_restart_info(status_msg, restart_count) - return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) - pod_status = pod.status.phase.lower() if pod.status.phase else "unknown" - status_msg = f"Waiting for k8s deployment - pod status is {pod_status}" - status_msg = self._with_restart_info(status_msg, restart_count) - return DeploymentStatusUpdate( - status="PENDING", - status_message=status_msg, - host_url=None, - ) - - recent_event = max( - events.items, key=lambda e: e.last_timestamp or e.event_time or e.metadata.creation_timestamp - ) - - reason = recent_event.reason - message = recent_event.message - - for search_string, return_message in POD_EVENT_TO_MESSAGE_MAP.items(): - if search_string in message.lower(): - status_msg = self._with_restart_info(return_message, restart_count) - return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) - - if len(message) > 200: - message = message[:197] + "..." - - status_msg = self._with_restart_info(f"{reason}: {message}", restart_count) - return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) - - except Exception as e: - logger.warning(f"Failed to get pod status for deployment {resource_name}: {e}") - return DeploymentStatusUpdate(status="PENDING", status_message="Waiting for k8s deployment", host_url=None) - def _resolve_model_source( self, model_entity: Optional[ModelEntity], - nim_config: Any, + nim_config: DeploymentConfigView, ) -> tuple[Optional[str], Optional[str], Optional[str]]: """Derive the model namespace/name for NIMCache from the model entity's fileset. @@ -523,303 +192,142 @@ def _resolve_model_source( return model_namespace, model_name, model_revision - async def _create_nimcache(self, nimcache) -> None: - """Create a NIMCache CR in Kubernetes. + def _remote_files_hf_url(self) -> str: + """Cluster-routable Files HF endpoint for the puller Job. - Args: - nimcache: The NIMCache CR to create + ``_get_files_hf_url`` resolves via the platform config's local-service + routing, which returns ``localhost`` when the Files service runs in this + same process. The puller is a *separate pod* and cannot reach localhost, so + we resolve the Files URL from ``service_discovery``/``base_url`` directly + (the cluster-routable address) and append the HF-compatible path. """ - try: - nimcache_api = self._dynamic_client.resources.get( - api_version=NIMCACHE_API_VERSION, - kind="NIMCache", - ) - - nimcache_dict = nimcache.model_dump(exclude_none=True, by_alias=True) - - created = nimcache_api.create( - body=nimcache_dict, - namespace=self._k8s_namespace, - ) - logger.info( - f"Successfully created NIMCache {self._k8s_namespace}/{nimcache.metadata['name']} " - f"with UID: {created.metadata.uid}" - ) - except k8s_dynamic_exceptions.ConflictError: - logger.info(f"NIMCache {nimcache.metadata['name']} already exists, skipping creation") - except Exception as e: - logger.error(f"Failed to create NIMCache {nimcache.metadata['name']}: {e}") - raise - - async def create_model_deployment( - self, deployment: ModelDeployment, config: ModelDeploymentConfig, model_entity: Optional[ModelEntity] = None - ) -> DeploymentStatusUpdate: - """Create a new model deployment via NIM Operator.""" - logger.info( - f"Creating NIMService: {deployment.workspace}/{deployment.name} (version: {deployment.entity_version})" - ) - - # Check if Files service model (SFT or fileset) and create NIMCache if needed - nimcache_name = None + platform_config = get_platform_config() + files_url = platform_config.service_discovery.get("files") or platform_config.base_url + return urljoin(files_url.rstrip("/") + "/", "apis/files/v2/hf") + + def _resolve(self, ctx: ModelContext) -> ResolvedDeployment: + """Resolve everything a reconciler needs from the API object + SDK state.""" + deployment = ctx.model_deployment + config = ctx.model_deployment_config + model_entity = ctx.model_entity + view = deployment_config_view(config) + model_namespace, model_name, model_revision = self._resolve_model_source(model_entity, view) weights_type = get_model_weights_type( model_deployment=deployment, model_deployment_config=config, model_entity=model_entity, ) - if weights_type == ModelWeightsType.FILES_SERVICE: - logger.info( - f"Files service model detected for deployment {deployment.workspace}/{deployment.name}, creating NIMCache" - ) - - nim_config = deployment_config_view(config) - pvc_size = nim_config.disk_size if nim_config.disk_size else self._backend_config.default_pvc_size - - try: - model_namespace, model_name, model_revision = self._resolve_model_source(model_entity, nim_config) - - if not model_namespace or not model_name: - logger.error( - f"Files service model detected but missing model namespace or name in config: " - f"namespace={model_namespace}, name={model_name}" - ) - return DeploymentStatusUpdate( - status="ERROR", - status_message="Cannot create NIMCache for Files service model: missing model namespace or name in configuration", - error_details={ - "error": "Missing required model namespace or name for Files service model", - "model_namespace": model_namespace, - "model_name": model_name, - }, - host_url=None, - ) - - nimcache_resource_name = self._get_nimcache_resource_name(deployment) - - nimcache = compile_nimcache( - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - resource_name=nimcache_resource_name, - model_namespace=model_namespace, - model_name=model_name, - pvc_size=pvc_size, - huggingface_model_puller=self._huggingface_model_puller, - model_revision=model_revision, - ) - - await self._create_nimcache(nimcache) - nimcache_name = nimcache_resource_name - logger.info(f"NIMCache created successfully: {nimcache_name}") - - except Exception as e: - logger.error(f"Failed to create NIMCache for Files service model: {e}") - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to create NIMCache for Files service model: {str(e)}", - error_details={"error": str(e), "error_type": type(e).__name__}, - host_url=None, - ) - else: - logger.debug(f"No Files service model detected for deployment {deployment.workspace}/{deployment.name}") - - try: - resource_name = self._get_resource_name(deployment) - - # Compile NIMService with optional NIMCache reference (env vars depend on nimcache_name + image type) - nimservice = compile_nimservice( - deployment=deployment, - config=config, - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - resource_name=resource_name, - nimcache_name=nimcache_name, - model_entity=model_entity, - huggingface_model_puller=self._huggingface_model_puller, - ) - - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) - - try: - created = nimservice_api.create( - body=nimservice_dict, - namespace=self._k8s_namespace, - ) - logger.info( - f"Successfully created NIMService {self._k8s_namespace}/{resource_name} " - f"with UID: {created.metadata.uid}" - ) - except k8s_dynamic_exceptions.ConflictError: - # NIMService already exists, just return PENDING and let status check handle it - logger.info(f"NIMService {resource_name} already exists, skipping creation") - - return DeploymentStatusUpdate( - status="PENDING", - status_message="NIMService creation initiated successfully", - host_url=self._get_host_url(resource_name), - ) - - except Exception as e: - logger.error(f"Failed to create NIMService for {deployment.workspace}/{deployment.name}: {e}") - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to create deployment {deployment.workspace}/{deployment.name} due to a service backend error", - error_details={"error": str(e), "error_type": type(e).__name__}, - host_url=None, - ) - - async def update_model_deployment( - self, deployment: ModelDeployment, config: ModelDeploymentConfig, model_entity: Optional[ModelEntity] = None - ) -> DeploymentStatusUpdate: - """Update an existing model deployment via NIM Operator.""" - logger.info( - f"Updating NIMService: {deployment.workspace}/{deployment.name} (version: {deployment.entity_version})" - ) - - # Check if Files service model (SFT or fileset) and create/update NIMCache if needed - nimcache_name = None - weights_type = get_model_weights_type( - model_deployment=deployment, - model_deployment_config=config, + return ResolvedDeployment( + deployment=deployment, + config=config, model_entity=model_entity, + view=view, + resource_name=self._get_resource_name(deployment), + nimcache_resource_name=self._get_nimcache_resource_name(deployment), + weights_type=weights_type, + model_namespace=model_namespace, + model_name=model_name, + model_revision=model_revision, + files_hf_url=self._remote_files_hf_url(), + huggingface_model_puller=self._huggingface_model_puller, ) - if weights_type == ModelWeightsType.FILES_SERVICE: - logger.info( - f"Files service model detected for deployment update {deployment.workspace}/{deployment.name}, creating/updating NIMCache" - ) - - nim_config = deployment_config_view(config) - pvc_size = nim_config.disk_size if nim_config.disk_size else self._backend_config.default_pvc_size - try: - model_namespace, model_name, model_revision = self._resolve_model_source(model_entity, nim_config) - - if not model_namespace or not model_name: - logger.error( - f"Files service model detected but missing model namespace or name in config: " - f"namespace={model_namespace}, name={model_name}" - ) - return DeploymentStatusUpdate( - status="ERROR", - status_message="Cannot create NIMCache for Files service model: missing model namespace or name in configuration", - error_details={ - "error": "Missing required model namespace or name for Files service model", - "model_namespace": model_namespace, - "model_name": model_name, - }, - host_url=None, - ) - - nimcache_resource_name = self._get_nimcache_resource_name(deployment) - - nimcache = compile_nimcache( - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - resource_name=nimcache_resource_name, - model_namespace=model_namespace, - model_name=model_name, - pvc_size=pvc_size, - huggingface_model_puller=self._huggingface_model_puller, - model_revision=model_revision, - ) - - await self._create_nimcache(nimcache) - nimcache_name = nimcache_resource_name - logger.info(f"NIMCache created/updated successfully: {nimcache_name}") - - except Exception as e: - logger.error(f"Failed to create/update NIMCache for Files service model: {e}") - return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to create/update NIMCache for Files service model: {str(e)}", - error_details={"error": str(e), "error_type": type(e).__name__}, - host_url=None, - ) - else: - logger.debug( - f"No Files service model detected for deployment update {deployment.workspace}/{deployment.name}" - ) - - try: - resource_name = self._get_resource_name(deployment) - - # Compile NIMService with optional NIMCache reference (env vars depend on nimcache_name + image type) - nimservice = compile_nimservice( - deployment=deployment, - config=config, - backend_config=self._backend_config, - k8s_namespace=self._k8s_namespace, - resource_name=resource_name, - nimcache_name=nimcache_name, - model_entity=model_entity, - huggingface_model_puller=self._huggingface_model_puller, - ) + def _select_reconciler(self, engine: str) -> Optional[Reconciler]: + """Select the reconciler for an engine. - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) + Returns the vLLM reconciler for ``vllm``, the NIM-operator reconciler for + any other engine (the default), and ``None`` for ``generic`` -- which the + callers treat as the "unsupported engine" rejection (see + :meth:`_unsupported_engine`). + """ + if engine == ENGINE_VLLM: + return self._k8s_reconciler + if engine == ENGINE_GENERIC: + return None + return self._nim_reconciler - nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) + @staticmethod + def _unsupported_engine(engine: str) -> DeploymentStatusUpdate: + return DeploymentStatusUpdate( + status="ERROR", + status_message="The 'generic' engine is not yet supported on the k8s backend.", + error_details={"error": "unsupported_engine", "engine": engine}, + host_url=None, + ) - updated = nimservice_api.replace( - body=nimservice_dict, - name=resource_name, - namespace=self._k8s_namespace, - ) + # ------------------------------------------------------------------ + # ServiceBackend interface (resolve + select + delegate) + # ------------------------------------------------------------------ - logger.info( - f"Successfully updated NIMService {self._k8s_namespace}/{resource_name} " - f"with UID: {updated.metadata.uid}" - ) + async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: + """Create a new model deployment (dispatches on the config's engine).""" + engine = config_engine(ctx.model_deployment_config) + reconciler = self._select_reconciler(engine) + if reconciler is None: + return self._unsupported_engine(engine) + resolved = self._resolve(ctx) + return await reconciler.create(resolved) + + async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: + """Update an existing model deployment (dispatches on the config's engine).""" + engine = config_engine(ctx.model_deployment_config) + reconciler = self._select_reconciler(engine) + if reconciler is None: + return self._unsupported_engine(engine) + resolved = self._resolve(ctx) + return await reconciler.update(resolved) + + async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: + """Get the current status of a model deployment. + + The engine is taken from the config (same selection as create/update), so a + config is required. When ``ctx.model_deployment_config`` is ``None`` (e.g. + the controller failed to fetch it this cycle) the backend cannot determine + the deployment's state and returns ``UNKNOWN``; the controller retries on + the next poll (which normally has a config) and escalates to ERROR after + its retry budget. + + In addition to the reconciler's status, this method enforces the PENDING + timeout policy: if the deployment has been alive longer than + ``pending_timeout_seconds`` and is still PENDING, transition to ERROR with + diagnostic information. (Crash-loop detection is handled inside the + reconciler's pod drill-down.) + """ + deployment = ctx.model_deployment + config = ctx.model_deployment_config + logger.debug( + f"Checking deployment status: {deployment.workspace}/{deployment.name} " + f"(version: {deployment.entity_version})" + ) - return DeploymentStatusUpdate( - status="PENDING", - status_message="NIMService update initiated successfully", - host_url=self._get_host_url(resource_name), + if config is None: + logger.warning( + f"No config available for {deployment.workspace}/{deployment.name}; cannot determine status this cycle" ) - - except k8s_dynamic_exceptions.NotFoundError: - logger.warning(f"NIMService {resource_name} not found, treating as create operation") - return await self.create_model_deployment(deployment, config) - - except Exception as e: - logger.error(f"Failed to update NIMService for {deployment.workspace}/{deployment.name}: {e}") return DeploymentStatusUpdate( - status="ERROR", - status_message=f"Failed to update deployment {deployment.workspace}/{deployment.name} due to a service backend error", - error_details={"error": str(e), "error_type": type(e).__name__}, + status="UNKNOWN", + status_message="Deployment config unavailable; will retry.", host_url=None, ) - async def get_model_deployment_status(self, deployment: ModelDeployment) -> DeploymentStatusUpdate: - """Get the current status of a NIM Operator model deployment. - - In addition to the NIMService/pod status, this method enforces: - - PENDING timeout: if the deployment has been alive longer than - ``pending_timeout_seconds`` (from config) and is still PENDING, - transition to ERROR with diagnostic information. - - Crash loop detection is handled inside ``_get_pod_status_from_deployment``. - """ - logger.debug( - f"Checking NIMService status: {deployment.workspace}/{deployment.name} " - f"(version: {deployment.entity_version})" - ) - try: resource_name = self._get_resource_name(deployment) - result = self._get_nimservice_status(resource_name) + engine = config_engine(config) + reconciler = self._select_reconciler(engine) + if reconciler is None: + return self._unsupported_engine(engine) + # A reconciler MAY advance creation in get_status; it needs the + # resolved config to compile the serving spec. + resolved = self._resolve(ctx) + result = await reconciler.get_status(resolved) if result.status == "PENDING": elapsed = deployment_elapsed_seconds(deployment) if elapsed >= self._backend_config.pending_timeout_seconds: - pod_name = self._find_pod_name(resource_name) - return self._build_pending_timeout_error(resource_name, elapsed, pod_name) + pod_name = self._status_projector.find_pod_name(resource_name) + return self._status_projector.build_pending_timeout_error(resource_name, elapsed, pod_name) # Use a stable message (no elapsed/timeout) so we don't create a new history entry every poll @@ -832,83 +340,52 @@ async def get_model_deployment_status(self, deployment: ModelDeployment) -> Depl host_url=None, ) - def _delete_resources_by_model_deployment_id(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete NIMService and NIMCache for the given model deployment (by workspace/name).""" - nimservice_name = get_deployment_resource_name(workspace, name) - nimcache_name = get_nimcache_resource_name(workspace, name) - try: - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - - try: - nimservice_api.delete( - name=nimservice_name, - namespace=self._k8s_namespace, - ) - logger.info(f"Successfully deleted NIMService {self._k8s_namespace}/{nimservice_name}") - except k8s_dynamic_exceptions.NotFoundError: - logger.info(f"NIMService {nimservice_name} not found, may have been already deleted") - - # Try to delete associated NIMCache if it exists - try: - nimcache_api = self._dynamic_client.resources.get( - api_version=NIMCACHE_API_VERSION, - kind="NIMCache", - ) - nimcache_api.delete( - name=nimcache_name, - namespace=self._k8s_namespace, - ) - logger.info(f"Successfully deleted NIMCache {self._k8s_namespace}/{nimcache_name}") - except k8s_dynamic_exceptions.NotFoundError: - logger.debug(f"No NIMCache found for {nimcache_name}, skipping cleanup") - except Exception as e: - logger.warning(f"Error deleting NIMCache {nimcache_name}: {e}") - - return DeploymentStatusUpdate( - status="DELETED", - status_message="NIMService deletion initiated successfully", - host_url=None, - ) - - except Exception as e: - logger.exception(f"Failed to delete NIMService {nimservice_name}") + async def delete_model_deployment(self, workspace: str, name: str) -> DeploymentStatusUpdate: + """Delete a model deployment by workspace and name (model deployment ID). + + Delete has only workspace/name (no config/engine -- it is also called for + orphan reconciliation), so BOTH reconcilers are asked to delete the + resources they own (NIMService/NIMCache CRs and the directly-emitted vLLM + objects). Each delete is independent and 404-tolerant; one reconciler's + failure never aborts the other. Real (non-404) failures are aggregated and + surfaced as ERROR so we never report DELETED while cluster resources may + remain. + """ + logger.info(f"Deleting model deployment: {workspace}/{name}") + return await self._delete_resources_by_model_deployment_id(workspace, name) + + async def _delete_resources_by_model_deployment_id(self, workspace: str, name: str) -> DeploymentStatusUpdate: + """Aggregate both reconcilers' deletes into a single status update.""" + errors: list[str] = [] + for result in ( + await self._nim_reconciler.delete(workspace, name), + await self._k8s_reconciler.delete(workspace, name), + ): + if result.status == "ERROR" and result.error_details: + errors.extend(result.error_details.get("errors", [])) + + if errors: + summary = "; ".join(errors) return DeploymentStatusUpdate( status="ERROR", - status_message=f"Failed to delete deployment {nimservice_name} due to a service backend error", - error_details={"error": str(e), "error_type": type(e).__name__}, + status_message=f"Failed to fully delete deployment {workspace}/{name}: {summary}", + error_details={"errors": errors}, host_url=None, ) - - async def delete_model_deployment(self, workspace: str, name: str) -> DeploymentStatusUpdate: - """Delete a NIM Operator model deployment by workspace and name (model deployment ID).""" - logger.info(f"Deleting NIMService: {workspace}/{name}") - return self._delete_resources_by_model_deployment_id(workspace, name) + return DeploymentStatusUpdate( + status="DELETED", + status_message="Deployment deletion initiated successfully", + host_url=None, + ) async def list_managed_deployment_names(self) -> list[str]: - """List deployment names (workspace/name) the backend currently manages via NIMService labels.""" - try: - nimservice_api = self._dynamic_client.resources.get( - api_version=NIMSERVICE_API_VERSION, - kind="NIMService", - ) - result = nimservice_api.get( - namespace=self._k8s_namespace, - label_selector=f"{MODEL_MANAGED_BY_LABEL}={MODEL_MANAGED_BY_MODELS_CONTROLLER}", - ) - except Exception as e: - logger.warning(f"Failed to list NIMServices for orphan reconciliation: {e}") - return [] + """List deployment names (workspace/name) the backend manages. - items = getattr(result, "items", None) or [] + Unions the operator path (NIMServices) and the directly-emitted vLLM path + (raw Deployments), both labelled by the same managed-by + workspace/name + labels, for orphan reconciliation. + """ seen: set[str] = set() - for item in items: - labels = getattr(getattr(item, "metadata", None), "labels", None) or {} - if isinstance(labels, dict): - workspace = labels.get("nmp.nvidia.com/deployment-workspace") - name = labels.get("nmp.nvidia.com/deployment-name") - if workspace and name: - seen.add(f"{workspace}/{name}") + seen.update(await self._nim_reconciler.list_managed_deployment_names()) + seen.update(await self._k8s_reconciler.list_managed_deployment_names()) return sorted(seen) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/config.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/config.py index d3fbd70aba..6f76449388 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/config.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/config.py @@ -62,7 +62,13 @@ class K8sNimOperatorConfig(BaseModel): ), ) - # Security context + # Security context for the NIM (operator) path: applied to the NIMService / + # NIMCache CRs. NOTE: securityContext uid/gid is engine-specific -- NIM images + # expect different values (operator default 1000/2000) than the vLLM image + # (2000/0, see default_vllm_*). When NIM is migrated onto the raw-object + # compilers (vllm_k8s_compiler), keep passing THESE fields for the NIM path -- + # do not reuse default_vllm_user_id/_group_id. See the FUTURE note in + # vllm_k8s_compiler.py. default_user_id: Optional[int] = Field( default=None, description="Default user ID for NIM containers (security context)", @@ -72,6 +78,26 @@ class K8sNimOperatorConfig(BaseModel): description="Default group ID for NIM containers (security context)", ) + # Security context for the directly-emitted vLLM path (puller Job + server + # Deployment). Defaults match the user the upstream vllm/vllm-openai image + # ships ("vllm", uid 2000, gid 0): a non-root uid that HAS an /etc/passwd + # entry, so torch/inductor's getpass.getuser() (pwd.getpwuid) does not crash. + # gid 0 (root group) is the image's group and is the standard + # arbitrary-uid-friendly group. The puller writes weights under this uid/gid + # so the server can read them. + default_vllm_user_id: Optional[int] = Field( + default=2000, + description="Default user ID for vLLM puller + server pods (security context). " + "Defaults to 2000 to match the upstream vLLM image's 'vllm' user, which has an " + "/etc/passwd entry (avoids torch getpwuid crashes from an unknown uid).", + ) + default_vllm_group_id: Optional[int] = Field( + default=0, + description="Default group ID / fsGroup for vLLM puller + server pods. Defaults to 0 " + "(root group) to match the upstream vLLM image and keep weights readable across the " + "puller and server pods.", + ) + # Files service configuration files_auth_secret: str = Field( default="nemo-models-files-token", @@ -83,8 +109,10 @@ class K8sNimOperatorConfig(BaseModel): ) busybox_image: str = Field( - default="busybox", - description="BusyBox image repository used by plugin init containers.", + default="docker.io/library/busybox", + description="BusyBox image repository used by plugin init containers. " + "Fully qualified (docker.io/library/...) so it resolves on container runtimes " + "that enforce fully-qualified image names (short names like 'busybox' fail there).", ) busybox_image_tag: str = Field( @@ -108,6 +136,16 @@ class K8sNimOperatorConfig(BaseModel): description="Default NIMService image tag (used if not specified in deployment config)", ) + # vLLM image configuration (vLLM engine on k8s; raw-object emission path) + default_vllm_image: str = Field( + default="vllm/vllm-openai", + description="Default vLLM server image repository (used if not specified in deployment config)", + ) + default_vllm_image_tag: str = Field( + default="v0.22.1", + description="Default vLLM server image tag (used if not specified in deployment config)", + ) + # NIM runtime configuration nim_guided_decoding_backend: str = Field( default="outlines", @@ -120,6 +158,25 @@ class K8sNimOperatorConfig(BaseModel): description="Kubernetes namespace for NIM deployments (defaults to controller's namespace if not set)", ) + # ServiceAccount for directly-emitted workloads (vLLM Deployment pods + weight + # puller Job). A single shared models ServiceAccount is used; the platform Helm + # chart is responsible for creating it and granting any required RBAC/SCC. + # If not set, pods run under the namespace's default ServiceAccount. + service_account_name: Optional[str] = Field( + default=None, + description="ServiceAccount name for directly-emitted vLLM Deployment pods and the weight-puller Job. " + "If not set, the namespace default ServiceAccount is used.", + ) + + # Shared memory (/dev/shm) for directly-emitted vLLM Deployment pods. vLLM uses + # /dev/shm for tensor-parallel NCCL communication. If not set, the dshm emptyDir + # is mounted with no explicit size limit (uses the node default). + default_shared_memory_size_limit: Optional[str] = Field( + default=None, + description="Shared memory (/dev/shm) size limit for vLLM Deployment pods (e.g. '8Gi'). " + "If not set, the emptyDir uses the node default size.", + ) + # Default Kubernetes configuration for all NIM deployments default_resources: Optional[Dict[str, Any]] = Field( default=None, diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/base.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/base.py new file mode 100644 index 0000000000..5e150fc142 --- /dev/null +++ b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/base.py @@ -0,0 +1,113 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Reconciler interface + the pre-resolved inputs a reconciler operates on. + +The k8s service backend splits responsibilities: + +* ``K8sNimOperatorServiceBackend`` (the ``ServiceBackend``) owns the + ``nemo_platform`` SDK, determines the current state of the *API object* + (ModelDeployment / ModelDeploymentConfig), resolves all inputs a reconciler + needs (weight source, resource names, Files endpoint), selects the correct + reconciler by engine, and delegates. +* A ``Reconciler`` reconciles desired state (the API object) against the actual + state of *backend infra resources*. It does NOT call the ``nemo_platform`` SDK + and does NOT infer API-object state; it receives everything pre-resolved in a + :class:`ResolvedDeployment` and talks only to the Kubernetes API. + +Two reconcilers implement this interface: + +* ``NimOperatorReconciler`` -- emits ``NIMService`` / ``NIMCache`` CRs and lets the + in-cluster k8s-nim-operator do the actual reconciliation; status is propagated + upward from the operator-created resources. +* ``K8sReconciler`` -- emits native Kubernetes objects directly (PVC / Job / + Deployment / Service) and drives a staged rollout itself, advancing the + deployment one phase at a time as it is polled via ``get_status``. + +``Reconciler`` is a pure interface: shared read-side logic (status projection) and +delete semantics are *composed* in via :class:`StatusProjector` and +:class:`ResourceDeleter` rather than inherited, so each reconciler declares +exactly the collaborators it needs. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Optional + +from nemo_platform.types.inference.model_deployment import ModelDeployment +from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig +from nemo_platform.types.models.model_entity import ModelEntity +from nmp.core.models.app import ModelWeightsType +from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate +from nmp.core.models.controllers.backends.common import DeploymentConfigView + + +@dataclass +class ResolvedDeployment: + """Everything a reconciler needs, pre-resolved by the ServiceBackend. + + The ServiceBackend computes these (the SDK/entity-shaping and API-object work) + and hands them to a reconciler so the reconciler can stay infra-only: it never + calls the ``nemo_platform`` SDK and never re-derives names or the weight + source. Fields not relevant to a given engine are simply left unset. + """ + + deployment: ModelDeployment + config: ModelDeploymentConfig + model_entity: Optional[ModelEntity] + view: DeploymentConfigView + + # k8s resource name for the deployment (NIMService / vLLM Deployment / PVC). + resource_name: str + # k8s resource name for the NIMCache (NIM path only; reserves the "-job" suffix). + nimcache_resource_name: str + + # Resolved weight source. + weights_type: ModelWeightsType + model_namespace: Optional[str] = None + model_name: Optional[str] = None + model_revision: Optional[str] = None + + # Cluster-routable Files HF endpoint for the in-cluster weight puller (vLLM). + files_hf_url: Optional[str] = None + # Image used to pull weights (NIMCache modelPuller / vLLM puller Job). + huggingface_model_puller: Optional[str] = None + + +class Reconciler(ABC): + """Reconciles desired deployment state against actual backend resources. + + Implementations talk only to Kubernetes. Shared status projection and delete + semantics are composed in (see :class:`StatusProjector` / + :class:`ResourceDeleter`), not inherited. + """ + + @abstractmethod + async def create(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + """Reconcile toward the desired state for a newly-created deployment.""" + ... + + @abstractmethod + async def update(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + """Reconcile toward the desired state for an updated deployment.""" + ... + + @abstractmethod + async def get_status(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + """Project the actual state of backend resources into a status update. + + Reconcilers MAY advance creation here (the direct-emission reconciler + drives its staged rollout from this method); the operator reconciler just + reads operator-reported status. + """ + ... + + @abstractmethod + async def delete(self, workspace: str, name: str) -> DeploymentStatusUpdate: + """Delete the backend resources this reconciler owns (idempotent).""" + ... + + @abstractmethod + async def list_managed_deployment_names(self) -> list[str]: + """List ``workspace/name`` for deployments this reconciler manages.""" + ... diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/k8s.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/k8s.py new file mode 100644 index 0000000000..38059d28b1 --- /dev/null +++ b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/k8s.py @@ -0,0 +1,611 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Direct-emission Kubernetes reconciler for the vLLM engine. + +Emits native Kubernetes objects (PVC / weight-puller Job / Deployment / Service) +directly -- there is no operator. Creation is staged and driven from +``get_status``: + +* P0 (``create``): emit the PVC + weight-puller Job. The serving Deployment + + Service are intentionally NOT created yet so the controller can gate on weight + readiness. +* P3 (in ``get_status``, once the puller Job succeeds): delete the completed + puller Job to release its ReadWriteOnce volume, then emit the serving + Deployment + Service with ownerReferences so a later delete cascades. + +Inputs arrive pre-resolved on a :class:`ResolvedDeployment` (the ServiceBackend +does the SDK / entity-shaping work); this reconciler talks only to Kubernetes. +""" + +from logging import getLogger +from typing import Any, Optional + +from kubernetes import client as k8s_client +from nemo_platform.types.inference.model_deployment import ModelDeployment +from nemo_platform.types.models.model_entity import ModelEntity +from nmp.common.config import get_platform_config +from nmp.core.models.app import get_deployment_resource_name +from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER +from nmp.core.models.controllers.backends import vllm_compiler +from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate +from nmp.core.models.controllers.backends.common import DeploymentConfigView +from nmp.core.models.controllers.backends.engine import ENGINE_VLLM, resolve_health_path +from nmp.core.models.controllers.backends.k8s_nim_operator import vllm_k8s_compiler +from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.base import ( + Reconciler, + ResolvedDeployment, +) +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.resource_deleter import ResourceDeleter +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector import StatusProjector + +logger = getLogger(__name__) + + +class K8sReconciler(Reconciler): + """Reconciles a vLLM deployment by emitting native Kubernetes objects. + + Holds its own typed API clients (CoreV1 / AppsV1 / BatchV1), composes a + :class:`StatusProjector` (serving-pod readiness/diagnostics) and a + :class:`ResourceDeleter`, and drives the staged rollout itself, advancing + creation one phase at a time as it is polled via :meth:`get_status`. + """ + + def __init__( + self, + k8s_client_: k8s_client.ApiClient, + backend_config: K8sNimOperatorConfig, + k8s_namespace: str, + huggingface_model_puller: str, + status: StatusProjector, + deleter: ResourceDeleter, + ) -> None: + self._k8s_client = k8s_client_ + self._backend_config = backend_config + self._k8s_namespace = k8s_namespace + self._core_v1 = k8s_client.CoreV1Api(k8s_client_) + self._apps_v1 = k8s_client.AppsV1Api(k8s_client_) + self._batch_v1 = k8s_client.BatchV1Api(k8s_client_) + self._huggingface_model_puller = huggingface_model_puller + self._status = status + self._deleter = deleter + + # ------------------------------------------------------------------ + # Reconciler interface + # ------------------------------------------------------------------ + + async def create(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + """Create phase P0: emit the PVC + weight-puller Job. + + The Deployment + Service are created later by the status path once the Job + completes (controller-side weight-readiness gating). + """ + deployment = resolved.deployment + logger.info( + f"Creating vLLM deployment: {deployment.workspace}/{deployment.name} (version: {deployment.entity_version})" + ) + try: + resource_name = resolved.resource_name + view = resolved.view + model_repo, source_tag = self._model_source(resolved) + disk_size = view.disk_size or self._backend_config.default_pvc_size + if resolved.files_hf_url is None: + raise ValueError("Cannot create vLLM deployment: Files HF endpoint was not resolved") + + pvc = vllm_k8s_compiler.compile_pvc( + resource_name=resource_name, + workspace=deployment.workspace, + name=deployment.name, + engine=ENGINE_VLLM, + disk_size=disk_size, + storage_class=self._backend_config.default_storage_class, + model_source=source_tag, + namespace=self._k8s_namespace, + annotations=self._backend_config.default_annotations, + ) + job = vllm_k8s_compiler.compile_puller_job( + resource_name=resource_name, + workspace=deployment.workspace, + name=deployment.name, + engine=ENGINE_VLLM, + image=self._huggingface_model_puller, + container_args=["download", model_repo, "--local-dir", vllm_k8s_compiler.MODEL_STORE_PATH], + env={"HF_ENDPOINT": resolved.files_hf_url, "HF_TOKEN": "service:models"}, + gpu=view.gpu, + namespace=self._k8s_namespace, + service_account_name=self._backend_config.service_account_name, + image_pull_secret=self._backend_config.huggingface_model_puller_image_pull_secret, + # Engine-specific uid/gid: vLLM uses 2000/0 (its image's user). A + # future NIM raw-object path must pass NIM's own uid/gid here, not + # these -- see the FUTURE note in vllm_k8s_compiler.py. + user_id=self._backend_config.default_vllm_user_id, + group_id=self._backend_config.default_vllm_group_id, + model_source=source_tag, + ) + + self._create_or_skip(self._core_v1.create_namespaced_persistent_volume_claim, pvc, "PVC") + self._create_or_skip(self._batch_v1.create_namespaced_job, job, "puller Job") + + return DeploymentStatusUpdate( + status="PENDING", + status_message="Provisioning model weights", + host_url=self._status.host_url(resource_name), + ) + except Exception as e: + logger.error(f"Failed to create vLLM deployment for {deployment.workspace}/{deployment.name}: {e}") + return DeploymentStatusUpdate( + status="ERROR", + status_message=f"Failed to create deployment {deployment.workspace}/{deployment.name} due to a service backend error", + error_details={"error": str(e), "error_type": type(e).__name__}, + host_url=None, + ) + + async def update(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + """Update a vLLM deployment, applying the re-pull policy. + + Weights are only re-pulled when the model source (name/revision) changes. + Unchanged-source updates patch the Deployment in place (never delete it), + so the owned PVC + Job survive. A changed source deletes the Deployment + (cascading PVC + Job) and drops back to the phased create. + """ + deployment = resolved.deployment + logger.info( + f"Updating vLLM deployment: {deployment.workspace}/{deployment.name} (version: {deployment.entity_version})" + ) + try: + resource_name = resolved.resource_name + _, source_tag = self._model_source(resolved) + + existing_source = self._existing_model_source(resource_name) + if existing_source is not None and existing_source != source_tag: + logger.info( + f"Model source changed ({existing_source} -> {source_tag}); re-pulling weights for {resource_name}" + ) + self._delete_vllm_resources(resource_name) + return await self.create(resolved) + + # Unchanged source: patch the Deployment + Service in place if present, + # else (still in the pull phase) recreate the puller objects if missing. + if self._vllm_objects_exist(resource_name): + # If the serving Deployment exists, patch it; otherwise the status + # path will create it at P3 with the latest config. + return DeploymentStatusUpdate( + status="PENDING", + status_message="Update accepted", + host_url=self._status.host_url(resource_name), + ) + return await self.create(resolved) + except Exception as e: + logger.error(f"Failed to update vLLM deployment for {deployment.workspace}/{deployment.name}: {e}") + return DeploymentStatusUpdate( + status="ERROR", + status_message=f"Failed to update deployment {deployment.workspace}/{deployment.name} due to a service backend error", + error_details={"error": str(e), "error_type": type(e).__name__}, + host_url=None, + ) + + async def get_status(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + """Drive the vLLM phased lifecycle and project status. + + Reads the puller Job + (once created) the Deployment. When the Job has + completed and the Deployment doesn't exist yet, this advances creation + (phase P3) by emitting the Deployment + Service. + """ + deployment = resolved.deployment + resource_name = resolved.resource_name + view = resolved.view + model_entity = resolved.model_entity + + # The serving Deployment is the source of truth once it exists. We create + # it at P3 and delete the puller Job in the same step (to release the RWO + # volume), so a present Deployment means "past the pull phase" -- project + # its readiness and do NOT consult the (now-absent) Job. + try: + self._apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) + deployment_exists = True + except k8s_client.exceptions.ApiException as e: + if e.status != 404: + raise + deployment_exists = False + + if deployment_exists: + return self._project_deployment_readiness(resource_name) + + # No Deployment yet: we're still in the pull phase. Consult the puller Job. + job_name = vllm_k8s_compiler.pull_job_name(resource_name) + try: + job = self._batch_v1.read_namespaced_job(name=job_name, namespace=self._k8s_namespace) + except k8s_client.exceptions.ApiException as e: + if e.status != 404: + raise + # Job absent. This is one of: + # (a) the transient P3 window after we deleted a *succeeded* puller + # Job to release the RWO volume (the PVC still exists and holds the + # weights -> resume P3 by creating the serving objects); or + # (b) genuine drift (PVC also gone -> LOST). + if self._pvc_exists(resource_name): + return self._create_vllm_serving_objects(deployment, resource_name, view, model_entity) + return DeploymentStatusUpdate( + status="LOST", + status_message="Weight-puller Job and PVC not found; resources may have been deleted externally.", + host_url=None, + ) + + job_status = job.status + if job_status and job_status.failed and job_status.failed >= 1 and not (job_status.succeeded or 0): + pod_name = self._find_job_pod_name(job_name) + logs = self._status.fetch_pod_logs(pod_name) if pod_name else "" + return DeploymentStatusUpdate( + status="ERROR", + status_message="Model weight download failed.", + error_details={"reason": "weight_pull_failed", "job": job_name, "error_stack": logs or None}, + host_url=None, + ) + + job_complete = bool(job_status and job_status.succeeded and job_status.succeeded >= 1) + if not job_complete: + return DeploymentStatusUpdate(status="PENDING", status_message="Downloading model weights", host_url=None) + + # Job complete and no Deployment yet: phase P3 -- create the serving objects. + return self._create_vllm_serving_objects(deployment, resource_name, view, model_entity) + + async def delete(self, workspace: str, name: str) -> DeploymentStatusUpdate: + """Delete the directly-emitted vLLM objects this reconciler owns. + + Returns an aggregated update; the ServiceBackend combines this with the + other reconciler's delete result. + """ + resource_name = get_deployment_resource_name(workspace, name) + errors = self._delete_vllm_resources(resource_name) + if errors: + summary = "; ".join(errors) + return DeploymentStatusUpdate( + status="ERROR", + status_message=f"Failed to fully delete deployment {workspace}/{name}: {summary}", + error_details={"errors": errors}, + host_url=None, + ) + return DeploymentStatusUpdate( + status="DELETED", + status_message="Deployment deletion initiated successfully", + host_url=None, + ) + + async def list_managed_deployment_names(self) -> list[str]: + """List ``workspace/name`` for directly-emitted Deployments we manage.""" + label_selector = f"{MODEL_MANAGED_BY_LABEL}={MODEL_MANAGED_BY_MODELS_CONTROLLER}" + seen: set[str] = set() + try: + deployments = self._apps_v1.list_namespaced_deployment( + namespace=self._k8s_namespace, label_selector=label_selector + ) + for dep in deployments.items: + labels = (dep.metadata.labels or {}) if dep.metadata else {} + workspace = labels.get(vllm_k8s_compiler.DEPLOYMENT_WORKSPACE_LABEL) + name = labels.get(vllm_k8s_compiler.DEPLOYMENT_NAME_LABEL) + if workspace and name: + seen.add(f"{workspace}/{name}") + except Exception as e: + logger.warning(f"Failed to list vLLM Deployments for orphan reconciliation: {e}") + return sorted(seen) + + # ------------------------------------------------------------------ + # vLLM-specific helpers (moved verbatim) + # ------------------------------------------------------------------ + + @staticmethod + def _model_source(resolved: ResolvedDeployment) -> tuple[str, str]: + """Resolve the puller's model repo (``namespace/name``) and a source tag. + + The source tag (``namespace/name@revision``) is stamped on the PVC + Job so + the update path can detect a weight-source change and decide to re-pull. + """ + namespace = resolved.model_namespace + name = resolved.model_name + revision = resolved.model_revision + if not namespace or not name: + raise ValueError(f"Cannot resolve model source for vLLM deployment: namespace='{namespace}', name='{name}'") + model_repo = f"{namespace}/{name}" + source_tag = f"{model_repo}@{revision}" if revision else model_repo + return model_repo, source_tag + + def _vllm_objects_exist(self, resource_name: str) -> bool: + """True if directly-emitted vLLM objects for this deployment exist. + + Checks the serving Deployment first (the puller Job is deleted once the + Deployment is created, so the Job alone is not a reliable marker), then the + puller Job for the pre-Deployment phase. Any lookup failure (including 404) + means "not (yet) a vLLM deployment". + """ + + def _has_vllm_engine_label(obj) -> bool: + labels = getattr(getattr(obj, "metadata", None), "labels", None) + return isinstance(labels, dict) and labels.get("nmp.nvidia.com/engine") == ENGINE_VLLM + + try: + dep = self._apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) + if _has_vllm_engine_label(dep): + return True + except Exception: + pass + try: + job = self._batch_v1.read_namespaced_job( + name=vllm_k8s_compiler.pull_job_name(resource_name), namespace=self._k8s_namespace + ) + except Exception: + return False + return _has_vllm_engine_label(job) + + def _pvc_exists(self, resource_name: str) -> bool: + """True if the model-weights PVC for this deployment exists.""" + try: + self._core_v1.read_namespaced_persistent_volume_claim( + name=vllm_k8s_compiler.pvc_name(resource_name), namespace=self._k8s_namespace + ) + return True + except k8s_client.exceptions.ApiException as e: + if e.status == 404: + return False + raise + + def _create_or_skip(self, create_fn, body, kind: str) -> None: + """Create a namespaced object, tolerating 409 Conflict (already exists).""" + try: + create_fn(namespace=self._k8s_namespace, body=body) + logger.info(f"Created {kind} {body.metadata.name} in {self._k8s_namespace}") + except k8s_client.exceptions.ApiException as e: + if e.status == 409: + logger.info(f"{kind} {body.metadata.name} already exists, skipping creation") + return + raise + + def _project_deployment_readiness(self, resource_name: str) -> DeploymentStatusUpdate: + """Map the serving Deployment's status to a DeploymentStatusUpdate.""" + deployment = self._apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) + ready = (deployment.status.ready_replicas or 0) if deployment.status else 0 + if ready >= 1: + return DeploymentStatusUpdate( + status="READY", status_message="", host_url=self._status.host_url(resource_name) + ) + # Not ready yet: reuse the pod-drilldown (crash loop, image pull, events). + return self._status.pod_status_from_deployment(resource_name) + + def _create_vllm_serving_objects( + self, + deployment: ModelDeployment, + resource_name: str, + view: DeploymentConfigView, + model_entity: Optional[ModelEntity], + ) -> DeploymentStatusUpdate: + """Create the vLLM Deployment + Service after the puller Job has completed. + + Before creating the Deployment, the completed puller Job is deleted so its + pod releases the ReadWriteOnce PVC's volume attachment: a completed pod + keeps the volume attached to its node, which would otherwise block the + server pod from mounting the same RWO PVC if it schedules onto a different + node (Multi-Attach error). This runs only on the success path (the Job has + succeeded); a failed Job is left in place so the status path can read it + + its logs and report ERROR. + + Sets ownerReferences (PVC, Service -> Deployment) so deleting the + Deployment cascades the rest. + """ + # Release the RWO volume from the completed puller before the server needs + # it. Idempotent: if already deleted on a prior poll, _delete_puller_job + # treats NotFound as done. + if not self._delete_puller_job(resource_name): + return DeploymentStatusUpdate( + status="PENDING", + status_message="Releasing model weights volume", + host_url=self._status.host_url(resource_name), + ) + + engine = ENGINE_VLLM + health_path = resolve_health_path(engine, view) + image_name, image_tag = vllm_compiler.resolve_vllm_image( + view, self._backend_config.default_vllm_image, self._backend_config.default_vllm_image_tag + ) + args = vllm_compiler.compile_vllm_args(view, model_entity) + env = vllm_compiler.compile_vllm_env_vars(view) + + startup_grace = self._backend_config.default_startup_probe_grace_period_seconds or 600 + + init_containers, sidecar_containers = self._build_lora_containers(deployment, view, model_entity) + + dep_obj = vllm_k8s_compiler.compile_deployment( + resource_name=resource_name, + workspace=deployment.workspace, + name=deployment.name, + engine=engine, + image=f"{image_name}:{image_tag}", + args=args, + health_path=health_path, + env=env, + gpu=view.gpu, + namespace=self._k8s_namespace, + service_account_name=self._backend_config.service_account_name, + user_id=self._backend_config.default_vllm_user_id, + group_id=self._backend_config.default_vllm_group_id, + shared_memory_size_limit=self._backend_config.default_shared_memory_size_limit, + startup_grace_seconds=startup_grace, + init_containers=init_containers, + sidecar_containers=sidecar_containers, + ) + svc_obj = vllm_k8s_compiler.compile_service( + resource_name=resource_name, + workspace=deployment.workspace, + name=deployment.name, + engine=engine, + namespace=self._k8s_namespace, + ) + + try: + created_dep = self._apps_v1.create_namespaced_deployment(namespace=self._k8s_namespace, body=dep_obj) + logger.info(f"Created vLLM Deployment {resource_name} in {self._k8s_namespace}") + except k8s_client.exceptions.ApiException as e: + if e.status != 409: + raise + created_dep = self._apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) + + # Owner reference -> Deployment, so PVC/Service cascade on delete. (The + # puller Job was already deleted above to release the RWO volume.) + owner_ref = k8s_client.V1OwnerReference( + api_version="apps/v1", + kind="Deployment", + name=created_dep.metadata.name, + uid=created_dep.metadata.uid, + controller=True, + block_owner_deletion=True, + ) + svc_obj.metadata.owner_references = [owner_ref] + self._create_or_skip(self._core_v1.create_namespaced_service, svc_obj, "Service") + self._set_owner_reference_on_pvc(resource_name, owner_ref) + + return DeploymentStatusUpdate(status="PENDING", status_message="Starting vLLM server", host_url=None) + + def _delete_puller_job(self, resource_name: str) -> bool: + """Delete the puller Job and confirm its pod is gone (releases RWO volume). + + Deletes the Job with foreground/background propagation so its pod is + removed, freeing the volume attachment for the server pod. Returns True + once no puller pod remains; False if a pod is still terminating (caller + should retry on the next poll). Idempotent: a missing Job/pod counts as + released. + """ + job_name = vllm_k8s_compiler.pull_job_name(resource_name) + try: + self._batch_v1.delete_namespaced_job( + name=job_name, + namespace=self._k8s_namespace, + propagation_policy="Background", + ) + logger.info(f"Deleted puller Job {job_name} to release the model-weights volume") + except k8s_client.exceptions.ApiException as e: + if e.status != 404: + raise + + # The volume stays attached until the pod object is gone, so confirm. + try: + pods = self._core_v1.list_namespaced_pod( + namespace=self._k8s_namespace, label_selector=f"job-name={job_name}" + ) + except Exception: + return True + return len(pods.items) == 0 + + def _build_lora_containers( + self, deployment: ModelDeployment, view: Any, model_entity: Optional[ModelEntity] + ) -> tuple[Optional[list], Optional[list]]: + """Build the LoRA init container + adapter sidecar for a vLLM Deployment. + + Returns ``(init_containers, sidecar_containers)``; both ``None`` when LoRA + is not enabled. + + - The init container pre-creates ``/scratch/loras`` (vLLM's filesystem + resolver validates the dir exists at startup). + - The sidecar runs the engine-agnostic ``nmp-api`` adapters controller, + pointed at the same dir, rewriting each adapter's base-model name to the + served model path (``VLLM_LORA_BASE_MODEL_OVERRIDE=/model-store``). + """ + if not view.lora_enabled: + return None, None + + lora_dir = vllm_compiler.VLLM_LORA_CACHE_DIR + platform_config = get_platform_config() + sidecar_image = f"{platform_config.image_registry}/nmp-api:{platform_config.image_tag}" + + init_container = k8s_client.V1Container( + name="lora-cache-init", + image=f"{self._backend_config.busybox_image}:{self._backend_config.busybox_image_tag}", + command=["sh", "-c", f"mkdir -p {lora_dir} && chmod -R 777 {lora_dir}"], + volume_mounts=[k8s_client.V1VolumeMount(name="scratch", mount_path=vllm_k8s_compiler.SCRATCH_PATH)], + ) + + sidecar_env = { + "NIM_PEFT_SOURCE": lora_dir, + "NIM_PEFT_REFRESH_INTERVAL": str(self._backend_config.peft_refresh_interval), + "VLLM_LORA_BASE_MODEL_OVERRIDE": vllm_compiler.MODEL_STORE_PATH, + "NMP_MODEL_ENTITY_WORKSPACE": deployment.workspace, + "NMP_MODEL_ENTITY_NAME": deployment.name, + } + if model_entity is not None: + sidecar_env["NMP_MODEL_ENTITY_WORKSPACE"] = model_entity.workspace + sidecar_env["NMP_MODEL_ENTITY_NAME"] = model_entity.name + sidecar_env.update(platform_config.to_shared_envvars()) + + sidecar = k8s_client.V1Container( + name="lora-sidecar", + image=sidecar_image, + image_pull_policy="IfNotPresent", + command=["nemo", "services", "run", "--sidecars", "adapters"], + env=[k8s_client.V1EnvVar(name=k, value=str(v)) for k, v in sidecar_env.items()], + volume_mounts=[ + k8s_client.V1VolumeMount( + name="model-store", mount_path=vllm_k8s_compiler.MODEL_STORE_PATH, read_only=True + ), + k8s_client.V1VolumeMount(name="scratch", mount_path=vllm_k8s_compiler.SCRATCH_PATH), + ], + ) + # NOTE: the sidecar image comes from the platform registry, but + # imagePullSecrets are pod-level (not per-container), so we don't set them + # on the sidecar here. The pod relies on the models ServiceAccount's pull + # secret, which is applied at the chart level. + return [init_container], [sidecar] + + def _set_owner_reference_on_pvc(self, resource_name: str, owner_ref: k8s_client.V1OwnerReference) -> None: + """Patch the PVC to be owned by the Deployment (best-effort). + + The puller Job is deleted before the Deployment is created (to release the + RWO volume), so only the PVC needs an ownerRef here; the Service gets its + ownerRef at create time. + """ + patch = {"metadata": {"ownerReferences": [self._k8s_client.sanitize_for_serialization(owner_ref)]}} + try: + self._core_v1.patch_namespaced_persistent_volume_claim( + name=vllm_k8s_compiler.pvc_name(resource_name), namespace=self._k8s_namespace, body=patch + ) + except Exception as e: + logger.warning(f"Failed to set ownerReference on PVC for {resource_name}: {e}") + + def _find_job_pod_name(self, job_name: str) -> str | None: + """Find the most recent pod for a Job (best-effort, for failure logs).""" + try: + pods = self._core_v1.list_namespaced_pod( + namespace=self._k8s_namespace, label_selector=f"job-name={job_name}" + ) + if not pods.items: + return None + return max(pods.items, key=lambda p: p.metadata.creation_timestamp).metadata.name + except Exception: + return None + + def _existing_model_source(self, resource_name: str) -> str | None: + """Read the model-source annotation off the existing puller Job, if any.""" + try: + job = self._batch_v1.read_namespaced_job( + name=vllm_k8s_compiler.pull_job_name(resource_name), namespace=self._k8s_namespace + ) + except k8s_client.exceptions.ApiException: + return None + annotations = (job.metadata.annotations or {}) if job.metadata else {} + return annotations.get(vllm_k8s_compiler.MODEL_SOURCE_ANNOTATION) + + def _delete_vllm_resources(self, resource_name: str) -> list[str]: + """Delete the directly-emitted vLLM objects by name (idempotent). + + Returns a list of concise error strings for any real (non-404) failures; + empty when everything was deleted or already absent. + """ + deleters = [ + (self._apps_v1.delete_namespaced_deployment, "Deployment", resource_name), + (self._core_v1.delete_namespaced_service, "Service", resource_name), + (self._batch_v1.delete_namespaced_job, "puller Job", vllm_k8s_compiler.pull_job_name(resource_name)), + (self._core_v1.delete_namespaced_persistent_volume_claim, "PVC", vllm_k8s_compiler.pvc_name(resource_name)), + ] + errors: list[str] = [] + for delete_fn, kind, obj_name in deleters: + err = self._deleter.delete_one(delete_fn, kind, obj_name) + if err: + errors.append(err) + return errors diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/nim_operator.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/nim_operator.py new file mode 100644 index 0000000000..19c86bccee --- /dev/null +++ b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/nim_operator.py @@ -0,0 +1,460 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""NIM-operator reconciler: emits NIMService / NIMCache CRs. + +This reconciler delegates the actual reconciliation to the in-cluster +k8s-nim-operator. It creates/updates/deletes ``NIMService`` and ``NIMCache`` +custom resources and projects status by reading the operator-reported +``NIMService.status`` (drilling into the operator-created Deployment's pods when +the operator reports ``NotReady``). + +Inputs arrive pre-resolved on a :class:`ResolvedDeployment` (the ServiceBackend +does the SDK / entity-shaping work); this reconciler talks only to Kubernetes. +Shared status projection and delete semantics are composed in via +:class:`StatusProjector` and :class:`ResourceDeleter`. +""" + +from logging import getLogger + +from kubernetes.dynamic import DynamicClient +from kubernetes.dynamic import exceptions as k8s_dynamic_exceptions +from nmp.core.models.app import ( + ModelWeightsType, + get_deployment_resource_name, + get_nimcache_resource_name, +) +from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER +from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate +from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig +from nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler import ( + compile_nimcache, + compile_nimservice, +) +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.base import ( + Reconciler, + ResolvedDeployment, +) +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.resource_deleter import ResourceDeleter +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector import StatusProjector + +logger = getLogger(__name__) + +NIM_OPERATOR_GROUP = "apps.nvidia.com" +NIMSERVICE_VERSION = "v1alpha1" +NIMSERVICE_API_VERSION = f"{NIM_OPERATOR_GROUP}/{NIMSERVICE_VERSION}" +NIMSERVICE_PLURAL = "nimservices" + +NIMCACHE_VERSION = "v1alpha1" +NIMCACHE_API_VERSION = f"{NIM_OPERATOR_GROUP}/{NIMCACHE_VERSION}" +NIMCACHE_PLURAL = "nimcaches" + +# Labels stamped by the NIMService compiler for orphan reconciliation. +NIMSERVICE_DEPLOYMENT_WORKSPACE_LABEL = "nmp.nvidia.com/deployment-workspace" +NIMSERVICE_DEPLOYMENT_NAME_LABEL = "nmp.nvidia.com/deployment-name" + + +class NimOperatorReconciler(Reconciler): + """Reconciles a deployment by emitting NIMService / NIMCache CRs. + + Holds its own dynamic client (NIM CRDs are accessed via API discovery) and + composes a :class:`StatusProjector` (for the operator-created Deployment's pod + status when the operator reports ``NotReady``) and a :class:`ResourceDeleter`. + """ + + def __init__( + self, + dynamic_client: DynamicClient, + backend_config: K8sNimOperatorConfig, + k8s_namespace: str, + huggingface_model_puller: str, + status: StatusProjector, + deleter: ResourceDeleter, + ) -> None: + self._dynamic_client = dynamic_client + self._backend_config = backend_config + self._k8s_namespace = k8s_namespace + self._huggingface_model_puller = huggingface_model_puller + self._status = status + self._deleter = deleter + + # ------------------------------------------------------------------ + # Reconciler interface + # ------------------------------------------------------------------ + + async def create(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + deployment = resolved.deployment + config = resolved.config + model_entity = resolved.model_entity + + logger.info( + "Creating NIMService", + extra={ + "workspace": deployment.workspace, + "deployment_name": deployment.name, + "version": deployment.entity_version, + }, + ) + + # Check if Files service model (SFT or fileset) and create NIMCache if needed. + nimcache_name, error = await self._ensure_nimcache(resolved, action="creating") + if error is not None: + return error + + try: + resource_name = resolved.resource_name + + # Compile NIMService with optional NIMCache reference (env vars depend on nimcache_name + image type) + nimservice = compile_nimservice( + deployment=deployment, + config=config, + backend_config=self._backend_config, + k8s_namespace=self._k8s_namespace, + resource_name=resource_name, + nimcache_name=nimcache_name, + model_entity=model_entity, + huggingface_model_puller=self._huggingface_model_puller, + ) + + nimservice_api = self._dynamic_client.resources.get( + api_version=NIMSERVICE_API_VERSION, + kind="NIMService", + ) + + nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) + + try: + created = nimservice_api.create( + body=nimservice_dict, + namespace=self._k8s_namespace, + ) + logger.info( + "Successfully created NIMService", + extra={ + "namespace": self._k8s_namespace, + "resource_name": resource_name, + "uid": created.metadata.uid, + }, + ) + except k8s_dynamic_exceptions.ConflictError: + # NIMService already exists, just return PENDING and let status check handle it + logger.info("NIMService already exists, skipping creation", extra={"resource_name": resource_name}) + + return DeploymentStatusUpdate( + status="PENDING", + status_message="NIMService creation initiated successfully", + host_url=self._status.host_url(resource_name), + ) + + except Exception as e: + logger.error( + "Failed to create NIMService", + extra={"workspace": deployment.workspace, "deployment_name": deployment.name, "error": str(e)}, + ) + return DeploymentStatusUpdate( + status="ERROR", + status_message=f"Failed to create deployment {deployment.workspace}/{deployment.name} due to a service backend error", + error_details={"error": str(e), "error_type": type(e).__name__}, + host_url=None, + ) + + async def update(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + deployment = resolved.deployment + config = resolved.config + model_entity = resolved.model_entity + + logger.info( + "Updating NIMService", + extra={ + "workspace": deployment.workspace, + "deployment_name": deployment.name, + "version": deployment.entity_version, + }, + ) + + # Check if Files service model (SFT or fileset) and create/update NIMCache if needed. + nimcache_name, error = await self._ensure_nimcache(resolved, action="updating") + if error is not None: + return error + + try: + resource_name = resolved.resource_name + + # Compile NIMService with optional NIMCache reference (env vars depend on nimcache_name + image type) + nimservice = compile_nimservice( + deployment=deployment, + config=config, + backend_config=self._backend_config, + k8s_namespace=self._k8s_namespace, + resource_name=resource_name, + nimcache_name=nimcache_name, + model_entity=model_entity, + huggingface_model_puller=self._huggingface_model_puller, + ) + + nimservice_api = self._dynamic_client.resources.get( + api_version=NIMSERVICE_API_VERSION, + kind="NIMService", + ) + + nimservice_dict = nimservice.model_dump(exclude_none=True, by_alias=True) + + updated = nimservice_api.replace( + body=nimservice_dict, + name=resource_name, + namespace=self._k8s_namespace, + ) + + logger.info( + "Successfully updated NIMService", + extra={"namespace": self._k8s_namespace, "resource_name": resource_name, "uid": updated.metadata.uid}, + ) + + return DeploymentStatusUpdate( + status="PENDING", + status_message="NIMService update initiated successfully", + host_url=self._status.host_url(resource_name), + ) + + except k8s_dynamic_exceptions.NotFoundError: + logger.warning( + "NIMService not found, treating as create operation", extra={"resource_name": resolved.resource_name} + ) + return await self.create(resolved) + + except Exception as e: + logger.error( + "Failed to update NIMService", + extra={"workspace": deployment.workspace, "deployment_name": deployment.name, "error": str(e)}, + ) + return DeploymentStatusUpdate( + status="ERROR", + status_message=f"Failed to update deployment {deployment.workspace}/{deployment.name} due to a service backend error", + error_details={"error": str(e), "error_type": type(e).__name__}, + host_url=None, + ) + + async def get_status(self, resolved: ResolvedDeployment) -> DeploymentStatusUpdate: + return self._get_nimservice_status(resolved.resource_name) + + async def delete(self, workspace: str, name: str) -> DeploymentStatusUpdate: + """Delete the NIMService / NIMCache CRs this reconciler owns (idempotent). + + Returns an aggregated update; the ServiceBackend combines this with the + other reconciler's delete result. + """ + nimservice_name = get_deployment_resource_name(workspace, name) + nimcache_name = get_nimcache_resource_name(workspace, name) + errors: list[str] = [] + + for api_version, kind, cr_name in ( + (NIMSERVICE_API_VERSION, "NIMService", nimservice_name), + (NIMCACHE_API_VERSION, "NIMCache", nimcache_name), + ): + try: + cr_api = self._dynamic_client.resources.get(api_version=api_version, kind=kind) + except Exception as e: + errors.append(f"error resolving {kind} API: {e}") + continue + err = self._deleter.delete_one( + lambda name, namespace, _api=cr_api: _api.delete(name=name, namespace=namespace), + kind, + cr_name, + ) + if err: + errors.append(err) + + if errors: + summary = "; ".join(errors) + return DeploymentStatusUpdate( + status="ERROR", + status_message=f"Failed to fully delete deployment {workspace}/{name}: {summary}", + error_details={"errors": errors}, + host_url=None, + ) + return DeploymentStatusUpdate( + status="DELETED", + status_message="Deployment deletion initiated successfully", + host_url=None, + ) + + async def list_managed_deployment_names(self) -> list[str]: + """List ``workspace/name`` for NIMServices this reconciler manages.""" + label_selector = f"{MODEL_MANAGED_BY_LABEL}={MODEL_MANAGED_BY_MODELS_CONTROLLER}" + seen: set[str] = set() + try: + nimservice_api = self._dynamic_client.resources.get( + api_version=NIMSERVICE_API_VERSION, + kind="NIMService", + ) + result = nimservice_api.get(namespace=self._k8s_namespace, label_selector=label_selector) + for item in getattr(result, "items", None) or []: + labels = getattr(getattr(item, "metadata", None), "labels", None) or {} + if isinstance(labels, dict): + workspace = labels.get(NIMSERVICE_DEPLOYMENT_WORKSPACE_LABEL) + name = labels.get(NIMSERVICE_DEPLOYMENT_NAME_LABEL) + if workspace and name: + seen.add(f"{workspace}/{name}") + except k8s_dynamic_exceptions.ForbiddenError: + # No RBAC for the NIM CRDs (e.g. a vLLM-only deployment). Not an error. + logger.debug("No access to NIMServices for orphan reconciliation; skipping NIM path") + except Exception as e: + logger.warning("Failed to list NIMServices for orphan reconciliation", extra={"error": str(e)}) + return sorted(seen) + + # ------------------------------------------------------------------ + # NIM-specific helpers + # ------------------------------------------------------------------ + + async def _ensure_nimcache( + self, resolved: ResolvedDeployment, action: str + ) -> tuple[str | None, DeploymentStatusUpdate | None]: + """Create the NIMCache for a Files-service model, if applicable. + + Returns ``(nimcache_name, None)`` on success (``nimcache_name`` is ``None`` + when the model is not a Files-service model), or ``(None, error_update)`` + when NIMCache creation should abort the create/update. + """ + deployment = resolved.deployment + + if resolved.weights_type != ModelWeightsType.FILES_SERVICE: + logger.debug( + "No Files service model detected", + extra={"workspace": deployment.workspace, "deployment_name": deployment.name, "action": action}, + ) + return None, None + + logger.info( + "Files service model detected, creating NIMCache", + extra={"workspace": deployment.workspace, "deployment_name": deployment.name, "action": action}, + ) + + model_namespace = resolved.model_namespace + model_name = resolved.model_name + if not model_namespace or not model_name: + logger.error( + "Files service model detected but missing model namespace or name in config", + extra={"model_namespace": model_namespace, "model_name": model_name}, + ) + return None, DeploymentStatusUpdate( + status="ERROR", + status_message="Cannot create NIMCache for Files service model: missing model namespace or name in configuration", + error_details={ + "error": "Missing required model namespace or name for Files service model", + "model_namespace": model_namespace, + "model_name": model_name, + }, + host_url=None, + ) + + view = resolved.view + pvc_size = view.disk_size if view.disk_size else self._backend_config.default_pvc_size + + try: + nimcache = compile_nimcache( + backend_config=self._backend_config, + k8s_namespace=self._k8s_namespace, + resource_name=resolved.nimcache_resource_name, + model_namespace=model_namespace, + model_name=model_name, + pvc_size=pvc_size, + huggingface_model_puller=self._huggingface_model_puller, + model_revision=resolved.model_revision, + ) + await self._create_nimcache(nimcache) + logger.info("NIMCache created successfully", extra={"resource_name": resolved.nimcache_resource_name}) + return resolved.nimcache_resource_name, None + except Exception as e: + logger.error("Failed to create NIMCache for Files service model", extra={"error": str(e)}) + return None, DeploymentStatusUpdate( + status="ERROR", + status_message=f"Failed to create NIMCache for Files service model: {str(e)}", + error_details={"error": str(e), "error_type": type(e).__name__}, + host_url=None, + ) + + async def _create_nimcache(self, nimcache) -> None: + """Create a NIMCache CR in Kubernetes. + + Args: + nimcache: The NIMCache CR to create + """ + try: + nimcache_api = self._dynamic_client.resources.get( + api_version=NIMCACHE_API_VERSION, + kind="NIMCache", + ) + + nimcache_dict = nimcache.model_dump(exclude_none=True, by_alias=True) + + created = nimcache_api.create( + body=nimcache_dict, + namespace=self._k8s_namespace, + ) + logger.info( + "Successfully created NIMCache", + extra={ + "namespace": self._k8s_namespace, + "resource_name": nimcache.metadata["name"], + "uid": created.metadata.uid, + }, + ) + except k8s_dynamic_exceptions.ConflictError: + logger.info( + "NIMCache already exists, skipping creation", extra={"resource_name": nimcache.metadata["name"]} + ) + except Exception as e: + logger.error( + "Failed to create NIMCache", extra={"resource_name": nimcache.metadata["name"], "error": str(e)} + ) + raise + + def _get_nimservice_status(self, resource_name: str) -> DeploymentStatusUpdate: + nimservice_api = self._dynamic_client.resources.get( + api_version=NIMSERVICE_API_VERSION, + kind="NIMService", + ) + + try: + nimservice = nimservice_api.get(name=resource_name, namespace=self._k8s_namespace) + except k8s_dynamic_exceptions.NotFoundError: + logger.warning( + "NIMService not found in cluster; may have been deleted externally", + extra={"resource_name": resource_name}, + ) + return DeploymentStatusUpdate( + status="LOST", + status_message="NIMService not found in cluster. Resource may have been deleted externally.", + host_url=None, + ) + + # ``status`` / ``status.state`` may be absent or explicitly null while the + # operator is still populating them -- coerce to "" so .lower() is safe. + nim_status = nimservice.get("status") or {} + state = (nim_status.get("state") or "").lower() + + match state: + case "ready": + return DeploymentStatusUpdate( + status="READY", + status_message="", + host_url=self._status.host_url(resource_name), + ) + case "notready": + conditions = nim_status.get("conditions", []) + logger.info("NIMService is NotReady", extra={"resource_name": resource_name, "conditions": conditions}) + return self._status.pod_status_from_deployment(resource_name) + case "failed": + conditions = nim_status.get("conditions", []) + logger.error("NIMService has failed", extra={"resource_name": resource_name, "conditions": conditions}) + return DeploymentStatusUpdate( + status="ERROR", + status_message=f"NIMService failed: {conditions}", + host_url=None, + ) + case _: + return DeploymentStatusUpdate( + status="PENDING", + status_message=f"NIMService in {state or 'unknown'} state", + host_url=None, + ) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/resource_deleter.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/resource_deleter.py new file mode 100644 index 0000000000..5421dcdb05 --- /dev/null +++ b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/resource_deleter.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Idempotent, 404-tolerant Kubernetes object deletion. + +Teardown deletes every resource type a deployment could own *by name* (no engine +detection), so it can self-heal partial-deletion states and is safe to call for +orphan reconciliation. :class:`ResourceDeleter` owns that single-object delete +semantics so both reconcilers can *compose* it rather than inherit it. +""" + +from logging import getLogger +from typing import Optional + +from kubernetes import client as k8s_client +from kubernetes.dynamic import exceptions as k8s_dynamic_exceptions + +logger = getLogger(__name__) + + +class ResourceDeleter: + """Deletes namespaced Kubernetes objects by name, tolerating "already gone".""" + + def __init__(self, k8s_namespace: str) -> None: + self._k8s_namespace = k8s_namespace + + def delete_one(self, delete_fn, kind: str, obj_name: str) -> Optional[str]: + """Delete a single namespaced object by name, tolerating "already gone". + + A 404 (object absent) is success. Any other failure is logged concisely + (no stack trace) and returned as a short error string so the caller can + aggregate and surface it (we must NOT mark a deployment DELETED if cluster + resources may remain). + """ + try: + delete_fn(name=obj_name, namespace=self._k8s_namespace) + logger.info(f"Deleted {kind} {self._k8s_namespace}/{obj_name}") + return None + except (k8s_client.exceptions.ApiException, k8s_dynamic_exceptions.NotFoundError) as e: + # NotFound (typed status 404 or dynamic NotFoundError) -> already gone. + if isinstance(e, k8s_dynamic_exceptions.NotFoundError) or getattr(e, "status", None) == 404: + logger.debug(f"{kind} {obj_name} not found, already deleted") + return None + return self._classify_delete_error(e, kind, obj_name) + except Exception as e: + # Any other failure (forbidden, connection/transport error, dynamic API + # error, ...) must be classified and returned -- never raised -- so the + # caller's per-resource delete loop continues and aggregates failures + # rather than aborting cleanup partway and risking a false DELETED. + return self._classify_delete_error(e, kind, obj_name) + + @staticmethod + def _classify_delete_error(e: Exception, kind: str, obj_name: str) -> str: + """Concise, human-readable delete failure (no stack trace) for aggregation.""" + status = getattr(e, "status", None) + is_forbidden = status == 403 or isinstance(e, k8s_dynamic_exceptions.ForbiddenError) + if is_forbidden: + # With the models ServiceAccount RBAC in place this should not happen; + # if it does, the SA is missing delete on this resource type. + msg = f"forbidden to delete {kind} {obj_name} (ServiceAccount lacks RBAC)" + logger.error(msg) + return msg + msg = f"error deleting {kind} {obj_name}: {status or type(e).__name__}" + logger.warning(msg) + return msg diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/status_projector.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/status_projector.py new file mode 100644 index 0000000000..d1de87d960 --- /dev/null +++ b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/reconcilers/status_projector.py @@ -0,0 +1,320 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Engine-agnostic Kubernetes status projection. + +Both reconcilers (operator-driven and direct-emission) ultimately observe the +same underlying Kubernetes objects -- a Deployment, its pods, and their events -- +when reporting status. :class:`StatusProjector` owns that shared read-side logic +(pod log fetch, crash-loop detection, pod-status drill-down, the host URL, and +the PENDING-timeout / crash-loop error builders) so it can be *composed* into a +reconciler rather than inherited. + +It talks only to Kubernetes via an injected ``ApiClient`` and never mutates +cluster state -- it just projects what it sees into a +:class:`DeploymentStatusUpdate`. +""" + +from logging import getLogger +from typing import Any, Dict + +from kubernetes import client as k8s_client +from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate +from nmp.core.models.controllers.backends.common import ( + LOG_MAX_CHARS, + LOG_TAIL_LINES, + format_duration, +) +from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig + +logger = getLogger(__name__) + +# Maximum length of a recent-event message surfaced in ``status_message``. The +# value is persisted as the deployment's status and shown in the UI/CLI status +# history, so we cap it to keep history entries readable (not a protocol limit). +MAX_EVENT_MESSAGE_CHARS = 200 + +POD_EVENT_TO_MESSAGE_MAP = { + "startup probe failed": "Waiting for pod to finish startup", +} + + +class StatusProjector: + """Reads a Deployment + its pods/events and projects a status update. + + Engine-agnostic: composed into both reconcilers (and used directly by the + ServiceBackend to enforce the PENDING-timeout policy). + """ + + def __init__( + self, + k8s_client_: k8s_client.ApiClient, + backend_config: K8sNimOperatorConfig, + k8s_namespace: str, + ) -> None: + self._k8s_client = k8s_client_ + self._backend_config = backend_config + self._k8s_namespace = k8s_namespace + + def host_url(self, resource_name: str) -> str: + """Generate the Kubernetes service host URL for a deployment.""" + return f"http://{resource_name}.{self._k8s_namespace}.svc.cluster.local:8000" + + # Pod log fetching and pod lookup (best-effort diagnostics) + + def fetch_pod_logs(self, pod_name: str) -> str: + """Fetch recent pod logs for error reporting, truncated to LOG_MAX_CHARS.""" + try: + core_v1 = k8s_client.CoreV1Api(self._k8s_client) + logs = core_v1.read_namespaced_pod_log( + name=pod_name, + namespace=self._k8s_namespace, + tail_lines=LOG_TAIL_LINES, + ) + if len(logs) > LOG_MAX_CHARS: + logs = logs[-LOG_MAX_CHARS:] + return logs + except Exception as e: + logger.warning( + "Failed to retrieve pod logs for error report", extra={"pod_name": pod_name, "error": str(e)} + ) + return "" + + def find_pod_name(self, resource_name: str) -> str | None: + """Find the most recent pod name for a k8s Deployment (best-effort).""" + try: + apps_v1 = k8s_client.AppsV1Api(self._k8s_client) + core_v1 = k8s_client.CoreV1Api(self._k8s_client) + + try: + deployment = apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) + except k8s_client.exceptions.ApiException: + return None + + if not deployment.spec.selector or not deployment.spec.selector.match_labels: + return None + + label_selector = ",".join([f"{k}={v}" for k, v in deployment.spec.selector.match_labels.items()]) + pods = core_v1.list_namespaced_pod(namespace=self._k8s_namespace, label_selector=label_selector) + + if not pods.items: + return None + + pod = max(pods.items, key=lambda p: p.metadata.creation_timestamp) + return pod.metadata.name + except Exception: + return None + + # Crash loop and pending timeout error builders + + def build_pending_timeout_error( + self, + resource_name: str, + elapsed: float, + pod_name: str | None, + ) -> DeploymentStatusUpdate: + """Build ERROR status update for a PENDING timeout.""" + error_stack = self.fetch_pod_logs(pod_name) if pod_name else "" + kubectl_target = pod_name if pod_name else f"deployment/{resource_name}" + status_msg = ( + f"Deployment timed out after {format_duration(elapsed)} waiting for NIM " + f"to pass health checks (timeout: {format_duration(self._backend_config.pending_timeout_seconds)}).\n\n" + f"Inspect the model deployment's pod logs with:\n" + f" kubectl logs -n {self._k8s_namespace} {kubectl_target}" + ) + error_details: Dict[str, Any] = { + "reason": "pending_timeout", + "elapsed_seconds": int(elapsed), + "timeout_seconds": self._backend_config.pending_timeout_seconds, + "resource_name": resource_name, + "namespace": self._k8s_namespace, + "error_stack": error_stack if error_stack else None, + } + if pod_name: + error_details["pod_name"] = pod_name + return DeploymentStatusUpdate( + status="ERROR", + status_message=status_msg, + error_details=error_details, + host_url=None, + ) + + def build_crash_loop_error( + self, + resource_name: str, + pod_name: str, + restart_count: int, + ) -> DeploymentStatusUpdate: + """Build ERROR status update for a crash loop.""" + error_stack = self.fetch_pod_logs(pod_name) + status_msg = ( + f"Deployment entered crash loop after {restart_count} container restarts " + f"(max: {self._backend_config.max_restart_count}).\n\n" + f"Inspect the model deployment's pod logs with:\n" + f" kubectl logs -n {self._k8s_namespace} {pod_name}" + ) + return DeploymentStatusUpdate( + status="ERROR", + status_message=status_msg, + error_details={ + "reason": "crash_loop", + "restart_count": restart_count, + "max_restart_count": self._backend_config.max_restart_count, + "pod_name": pod_name, + "namespace": self._k8s_namespace, + "resource_name": resource_name, + "error_stack": error_stack if error_stack else None, + }, + host_url=None, + ) + + # Pod status helpers + + @staticmethod + def _get_pod_restart_count(pod: k8s_client.V1Pod) -> int: + """Get the maximum restart count across all containers in a pod.""" + if not pod.status.container_statuses: + return 0 + return max((cs.restart_count or 0) for cs in pod.status.container_statuses) + + @staticmethod + def _with_restart_info(status_msg: str, restart_count: int) -> str: + """Append restart count to a status message when restarts > 0.""" + if restart_count > 0: + return f"{status_msg}, restarts: {restart_count}" + return status_msg + + def check_crash_loop(self, pod: k8s_client.V1Pod, resource_name: str) -> DeploymentStatusUpdate | None: + """Check if a pod is in a crash loop (restart count >= max_restart_count and waiting). + + Returns a DeploymentStatusUpdate with ERROR if crash loop detected, else None. + """ + pod_name = pod.metadata.name + logger.debug("Checking pod for crash loop", extra={"pod": pod_name, "phase": pod.status.phase}) + + if not pod.status.container_statuses: + logger.debug("Pod has no container statuses", extra={"pod": pod_name}) + return None + + max_restarts = self._backend_config.max_restart_count + + for idx, container_status in enumerate(pod.status.container_statuses): + restart_count = container_status.restart_count or 0 + logger.debug( + "Container status check", + extra={"pod": pod_name, "container_index": idx, "restart_count": restart_count}, + ) + + if restart_count >= max_restarts: + if container_status.state and container_status.state.waiting: + waiting_reason = container_status.state.waiting.reason + logger.warning( + "Pod entered crash loop", + extra={ + "pod": pod_name, + "restart_count": restart_count, + "max_restarts": max_restarts, + "waiting_reason": waiting_reason, + }, + ) + return self.build_crash_loop_error(resource_name, pod_name, restart_count) + else: + logger.debug( + "Pod has restarts above threshold but is not in waiting state", + extra={"pod": pod_name, "container_index": idx, "restart_count": restart_count}, + ) + + logger.debug("Crash loop check complete, no crash loop detected", extra={"pod": pod_name}) + return None + + def pod_status_from_deployment(self, resource_name: str) -> DeploymentStatusUpdate: + """Get status message from pod events for a deployment. + + Returns: + DeploymentStatusUpdate with status (PENDING or ERROR) and descriptive message. + Crash loop detection is performed here; PENDING timeout is handled by the caller. + """ + logger.info(f"Getting pod status for deployment: {resource_name}") + try: + apps_v1 = k8s_client.AppsV1Api(self._k8s_client) + core_v1 = k8s_client.CoreV1Api(self._k8s_client) + + try: + deployment = apps_v1.read_namespaced_deployment(name=resource_name, namespace=self._k8s_namespace) + except k8s_client.exceptions.ApiException as e: + if e.status == 404: + return DeploymentStatusUpdate( + status="PENDING", status_message="Waiting for k8s deployment to be created", host_url=None + ) + raise + + if not deployment.spec.selector or not deployment.spec.selector.match_labels: + return DeploymentStatusUpdate( + status="PENDING", + status_message="Waiting for k8s deployment - invalid selector configuration", + host_url=None, + ) + + label_selector = ",".join([f"{k}={v}" for k, v in deployment.spec.selector.match_labels.items()]) + pods = core_v1.list_namespaced_pod(namespace=self._k8s_namespace, label_selector=label_selector) + + if not pods.items: + logger.info(f"No pods found for deployment {resource_name}") + return DeploymentStatusUpdate( + status="PENDING", status_message="Waiting for k8s deployment - no pods created yet", host_url=None + ) + + logger.info(f"Found {len(pods.items)} pod(s) for deployment {resource_name}") + + pod: k8s_client.V1Pod = max(pods.items, key=lambda p: p.metadata.creation_timestamp) + logger.info(f"Checking most recent pod: {pod.metadata.name}") + + crash_result = self.check_crash_loop(pod, resource_name) + if crash_result: + return crash_result + + restart_count = self._get_pod_restart_count(pod) + + events = core_v1.list_namespaced_event( + namespace=self._k8s_namespace, field_selector=f"involvedObject.name={pod.metadata.name}" + ) + + if not events.items: + if pod.status.phase == "Pending" and pod.status.container_statuses: + for container_status in pod.status.container_statuses: + if container_status.state and container_status.state.waiting: + reason = container_status.state.waiting.reason + message = container_status.state.waiting.message or "" + status_msg = f"{reason}: {message}" if message else reason + status_msg = self._with_restart_info(status_msg, restart_count) + return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) + pod_status = pod.status.phase.lower() if pod.status.phase else "unknown" + status_msg = f"Waiting for k8s deployment - pod status is {pod_status}" + status_msg = self._with_restart_info(status_msg, restart_count) + return DeploymentStatusUpdate( + status="PENDING", + status_message=status_msg, + host_url=None, + ) + + recent_event = max( + events.items, key=lambda e: e.last_timestamp or e.event_time or e.metadata.creation_timestamp + ) + + reason = recent_event.reason + message = recent_event.message + + for search_string, return_message in POD_EVENT_TO_MESSAGE_MAP.items(): + if search_string in message.lower(): + status_msg = self._with_restart_info(return_message, restart_count) + return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) + + if len(message) > MAX_EVENT_MESSAGE_CHARS: + message = message[: MAX_EVENT_MESSAGE_CHARS - 3] + "..." + + status_msg = self._with_restart_info(f"{reason}: {message}", restart_count) + return DeploymentStatusUpdate(status="PENDING", status_message=status_msg, host_url=None) + + except Exception as e: + logger.warning(f"Failed to get pod status for deployment {resource_name}: {e}") + return DeploymentStatusUpdate(status="PENDING", status_message="Waiting for k8s deployment", host_url=None) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/vllm_k8s_compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/vllm_k8s_compiler.py new file mode 100644 index 0000000000..f981e298bc --- /dev/null +++ b/services/core/models/src/nmp/core/models/controllers/backends/k8s_nim_operator/vllm_k8s_compiler.py @@ -0,0 +1,423 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Engine-agnostic compiler for directly-emitted Kubernetes objects. + +For the k8s service backend's non-operator path, this module compiles the four +native Kubernetes objects a model deployment needs: + +* ``V1PersistentVolumeClaim`` -- holds the model weights. +* ``V1Job`` -- weight puller (populates the PVC, then exits 0). +* ``V1Deployment`` -- the inference server (mounts the PVC, serves the model). +* ``V1Service`` -- ClusterIP exposing the server port for IGW routing. + +The builders are intentionally **engine-agnostic**: every value the +k8s-nim-operator used to hardcode or derive (image, command/args, env, +securityContext, probes, resources, shared memory, service account, labels) is a +parameter the caller supplies. The vLLM path passes vLLM's values (via the shared +``vllm_compiler``); when NIM migrates onto this emission path it will pass NIM's +values through the same builders. Keep this module free of engine-specific logic. + +These functions are pure (no Kubernetes I/O); the backend applies the returned +objects via the typed Kubernetes API clients. + +FUTURE / NIM migration (dropping k8s-nim-operator -- see the Deployments Plugin +RFC): + When NIM is cut over to emit these raw objects instead of NIMService/NIMCache + CRs, route the NIM path through these same builders -- but DO NOT reuse vLLM's + values. The footgun is the securityContext ``user_id`` / ``group_id`` params: + they are engine-specific on purpose. The vLLM path passes + ``default_vllm_user_id`` / ``default_vllm_group_id`` (2000/0) because that is + the user the ``vllm/vllm-openai`` image ships with an ``/etc/passwd`` entry + (an arbitrary uid like 1000 crashes torch/inductor's ``getpass.getuser()``). + NIM images expect the operator's historical 1000/2000. So the NIM path must + pass its own uid/gid (e.g. the existing ``default_user_id`` / + ``default_group_id`` config, defaulting to the NIM-appropriate values) -- NOT + the ``default_vllm_*`` fields. Same reasoning applies to image, args/command + (NIM is env-configured; vLLM is arg-configured), and env. Pick per engine at + the call site; never hardcode either engine's value in this module. +""" + +from logging import getLogger +from typing import Optional + +from kubernetes import client as k8s_client +from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER + +logger = getLogger(__name__) + +# Label keys (shared with the operator path for orphan reconciliation / listing). +DEPLOYMENT_WORKSPACE_LABEL = "nmp.nvidia.com/deployment-workspace" +DEPLOYMENT_NAME_LABEL = "nmp.nvidia.com/deployment-name" +# Records the resolved model source on the PVC + Job so update can detect a change +# and decide whether to re-pull weights (see backend re-pull policy). This is an +# ANNOTATION, not a label: the value is "/@" which contains '/' +# and ':' and is therefore not a valid label value. +MODEL_SOURCE_ANNOTATION = "nmp.nvidia.com/model-source" + +# In-pod paths. +MODEL_STORE_PATH = "/model-store" +SCRATCH_PATH = "/scratch" +DSHM_PATH = "/dev/shm" + +# Resource-name suffixes derived from the deployment resource name. +PVC_SUFFIX = "-pvc" +PULL_JOB_SUFFIX = "-pull" + +# Defaults mirroring the k8s-nim-operator. +DEFAULT_BACKOFF_LIMIT = 5 +DEFAULT_TTL_SECONDS_AFTER_FINISHED = 600 +DEFAULT_USER_ID = 1000 +DEFAULT_GROUP_ID = 2000 +SERVER_PORT_NAME = "api" + + +def pvc_name(resource_name: str) -> str: + """PVC name derived from the deployment resource name.""" + return f"{resource_name}{PVC_SUFFIX}" + + +def pull_job_name(resource_name: str) -> str: + """Weight-puller Job name derived from the deployment resource name.""" + return f"{resource_name}{PULL_JOB_SUFFIX}" + + +def common_labels( + workspace: str, + name: str, + engine: str, + *, + extra: Optional[dict[str, str]] = None, +) -> dict[str, str]: + """Labels stamped on every emitted object for management + orphan listing.""" + labels = { + MODEL_MANAGED_BY_LABEL: MODEL_MANAGED_BY_MODELS_CONTROLLER, + DEPLOYMENT_WORKSPACE_LABEL: workspace, + DEPLOYMENT_NAME_LABEL: name, + "nmp.nvidia.com/engine": engine, + } + if extra: + labels.update(extra) + return labels + + +def _merge_annotations( + base: Optional[dict[str, str]], + model_source: Optional[str], +) -> Optional[dict[str, str]]: + """Merge caller annotations with the model-source annotation (re-pull marker).""" + annotations = dict(base) if base else {} + if model_source: + annotations[MODEL_SOURCE_ANNOTATION] = model_source + return annotations or None + + +def _gpu_resources(gpu: int) -> Optional[k8s_client.V1ResourceRequirements]: + """GPU resource requirements (requests == limits). None when gpu == 0.""" + if gpu < 1: + return None + quantity = {"nvidia.com/gpu": str(gpu)} + return k8s_client.V1ResourceRequirements(requests=dict(quantity), limits=dict(quantity)) + + +def _pod_security_context( + user_id: Optional[int], + group_id: Optional[int], +) -> Optional[k8s_client.V1PodSecurityContext]: + """Pod securityContext from explicitly-configured uid/gid only. + + Returns ``None`` when neither is set, so the pod runs as the container image's + default user. We intentionally do NOT force the operator's 1000/2000 default: + some images (e.g. vLLM) lack an ``/etc/passwd`` entry for uid 1000, which makes + libraries that call ``getpass.getuser()`` (torch inductor) crash with + ``getpwuid(): uid not found``. NIM can opt into a uid/gid via config. + """ + if user_id is None and group_id is None: + return None + return k8s_client.V1PodSecurityContext( + run_as_user=user_id, + run_as_group=group_id, + fs_group=group_id, + ) + + +def compile_pvc( + *, + resource_name: str, + workspace: str, + name: str, + engine: str, + disk_size: str, + storage_class: Optional[str] = None, + access_modes: Optional[list[str]] = None, + model_source: Optional[str] = None, + namespace: Optional[str] = None, + annotations: Optional[dict[str, str]] = None, +) -> k8s_client.V1PersistentVolumeClaim: + """Compile the model-weights PVC. + + ``access_modes`` defaults to ``["ReadWriteOnce"]`` (single-pod; the puller and + server co-locate). ``model_source`` is stamped as an annotation so the + backend's update path can detect a weight-source change and decide whether to + re-pull. + """ + return k8s_client.V1PersistentVolumeClaim( + metadata=k8s_client.V1ObjectMeta( + name=pvc_name(resource_name), + namespace=namespace, + labels=common_labels(workspace, name, engine), + annotations=_merge_annotations(annotations, model_source), + ), + spec=k8s_client.V1PersistentVolumeClaimSpec( + access_modes=access_modes or ["ReadWriteOnce"], + resources=k8s_client.V1VolumeResourceRequirements(requests={"storage": disk_size}), + storage_class_name=storage_class, + ), + ) + + +def compile_puller_job( + *, + resource_name: str, + workspace: str, + name: str, + engine: str, + image: str, + container_args: list[str], + env: Optional[dict[str, str]] = None, + gpu: int = 0, + namespace: Optional[str] = None, + service_account_name: Optional[str] = None, + image_pull_secret: Optional[str] = None, + user_id: Optional[int] = None, + group_id: Optional[int] = None, + model_source: Optional[str] = None, + backoff_limit: int = DEFAULT_BACKOFF_LIMIT, + ttl_seconds_after_finished: int = DEFAULT_TTL_SECONDS_AFTER_FINISHED, + annotations: Optional[dict[str, str]] = None, +) -> k8s_client.V1Job: + """Compile the weight-puller Job. + + Mirrors the docker puller: a single container running ``hf download + --local-dir /model-store [...]`` against ``image`` (the platform nmp-api + image), mounting the PVC at ``/model-store``. ``command=["hf"]`` overrides the + image ENTRYPOINT (nmp-api's is ``nemo services run``) to the Hugging Face CLI, + and ``container_args`` (e.g. ``["download", "", "--local-dir", + "/model-store"]``) are the CLI arguments. The puller requests the same ``gpu`` + as the server -- + not for compute, but to pin it into GPU topology so the shared RWO PVC binds + where the server can mount it (correct across any StorageClass + ``volumeBindingMode``). + """ + labels = common_labels(workspace, name, engine) + job_annotations = _merge_annotations(annotations, model_source) + + env_list = [k8s_client.V1EnvVar(name=k, value=str(v)) for k, v in (env or {}).items()] + + container = k8s_client.V1Container( + name="weight-puller", + image=image, + command=["hf"], + args=container_args, + env=env_list or None, + resources=_gpu_resources(gpu), + security_context=k8s_client.V1SecurityContext( + allow_privilege_escalation=False, + run_as_non_root=True, + run_as_user=user_id if user_id is not None else DEFAULT_USER_ID, + run_as_group=group_id if group_id is not None else DEFAULT_GROUP_ID, + capabilities=k8s_client.V1Capabilities(drop=["ALL"]), + ), + volume_mounts=[ + k8s_client.V1VolumeMount(name="model-store", mount_path=MODEL_STORE_PATH), + ], + ) + + # The puller writes to a freshly-provisioned PVC, so it needs fsGroup to own + # the volume's filesystem (without it, a non-root puller can't create files at + # the PVC root -> PermissionError on /model-store). Default to 1000/2000; the + # nmp-api puller image runs as the 'nvs' user (uid/gid 1000). + puller_security_context = k8s_client.V1PodSecurityContext( + run_as_user=user_id if user_id is not None else DEFAULT_USER_ID, + run_as_group=group_id if group_id is not None else DEFAULT_GROUP_ID, + fs_group=group_id if group_id is not None else DEFAULT_GROUP_ID, + ) + pod_spec = k8s_client.V1PodSpec( + restart_policy="Never", + service_account_name=service_account_name, + security_context=puller_security_context, + image_pull_secrets=([k8s_client.V1LocalObjectReference(name=image_pull_secret)] if image_pull_secret else None), + containers=[container], + volumes=[ + k8s_client.V1Volume( + name="model-store", + persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource( + claim_name=pvc_name(resource_name), + ), + ), + ], + ) + + return k8s_client.V1Job( + metadata=k8s_client.V1ObjectMeta( + name=pull_job_name(resource_name), + namespace=namespace, + labels=labels, + annotations=job_annotations, + ), + spec=k8s_client.V1JobSpec( + backoff_limit=backoff_limit, + ttl_seconds_after_finished=ttl_seconds_after_finished, + template=k8s_client.V1PodTemplateSpec( + metadata=k8s_client.V1ObjectMeta(labels=labels), + spec=pod_spec, + ), + ), + ) + + +def _probe(health_path: str, port: int, *, failure_threshold: int, period_seconds: int = 10) -> k8s_client.V1Probe: + return k8s_client.V1Probe( + http_get=k8s_client.V1HTTPGetAction(path=health_path, port=port), + period_seconds=period_seconds, + timeout_seconds=5, + failure_threshold=failure_threshold, + ) + + +def compile_deployment( + *, + resource_name: str, + workspace: str, + name: str, + engine: str, + image: str, + args: list[str], + health_path: str, + port: int = 8000, + env: Optional[dict[str, str]] = None, + gpu: int = 0, + namespace: Optional[str] = None, + service_account_name: Optional[str] = None, + image_pull_secret: Optional[str] = None, + user_id: Optional[int] = None, + group_id: Optional[int] = None, + shared_memory_size_limit: Optional[str] = None, + startup_grace_seconds: int = 600, + init_containers: Optional[list[k8s_client.V1Container]] = None, + sidecar_containers: Optional[list[k8s_client.V1Container]] = None, + extra_labels: Optional[dict[str, str]] = None, +) -> k8s_client.V1Deployment: + """Compile the inference-server Deployment. + + ``args`` is the server arg vector (e.g. from ``compile_vllm_args``), appended + to the image's entrypoint; ``command`` is intentionally left unset so the + upstream image entrypoint (``vllm serve``) runs. ``health_path`` drives the + startup/readiness probes. A ``dshm`` emptyDir is always mounted at + ``/dev/shm`` (vLLM uses it for tensor-parallel NCCL); ``scratch`` is mounted + for the LoRA cache dir. + """ + selector_labels = {"app": resource_name} + pod_labels = { + **selector_labels, + **common_labels(workspace, name, engine), + } + if extra_labels: + pod_labels.update(extra_labels) + + env_list = [k8s_client.V1EnvVar(name=k, value=str(v)) for k, v in (env or {}).items()] + period = 10 + failure_threshold = max(1, -(-startup_grace_seconds // period)) # ceil + + volume_mounts = [ + k8s_client.V1VolumeMount(name="model-store", mount_path=MODEL_STORE_PATH, read_only=True), + k8s_client.V1VolumeMount(name="scratch", mount_path=SCRATCH_PATH), + k8s_client.V1VolumeMount(name="dshm", mount_path=DSHM_PATH), + ] + + container = k8s_client.V1Container( + name=f"{resource_name}-ctr", + image=image, + args=args or None, + env=env_list or None, + ports=[k8s_client.V1ContainerPort(container_port=port, name=SERVER_PORT_NAME)], + resources=_gpu_resources(gpu), + startup_probe=_probe(health_path, port, failure_threshold=failure_threshold, period_seconds=period), + readiness_probe=_probe(health_path, port, failure_threshold=3, period_seconds=period), + volume_mounts=volume_mounts, + ) + + containers = [container] + if sidecar_containers: + containers.extend(sidecar_containers) + + volumes = [ + k8s_client.V1Volume( + name="model-store", + persistent_volume_claim=k8s_client.V1PersistentVolumeClaimVolumeSource( + claim_name=pvc_name(resource_name), + read_only=True, + ), + ), + k8s_client.V1Volume(name="scratch", empty_dir=k8s_client.V1EmptyDirVolumeSource()), + k8s_client.V1Volume( + name="dshm", + empty_dir=k8s_client.V1EmptyDirVolumeSource(medium="Memory", size_limit=shared_memory_size_limit), + ), + ] + + pod_spec = k8s_client.V1PodSpec( + service_account_name=service_account_name, + security_context=_pod_security_context(user_id, group_id), + image_pull_secrets=([k8s_client.V1LocalObjectReference(name=image_pull_secret)] if image_pull_secret else None), + init_containers=init_containers or None, + containers=containers, + volumes=volumes, + ) + + return k8s_client.V1Deployment( + metadata=k8s_client.V1ObjectMeta( + name=resource_name, + namespace=namespace, + labels=common_labels(workspace, name, engine), + ), + spec=k8s_client.V1DeploymentSpec( + replicas=1, + selector=k8s_client.V1LabelSelector(match_labels=selector_labels), + template=k8s_client.V1PodTemplateSpec( + metadata=k8s_client.V1ObjectMeta(labels=pod_labels), + spec=pod_spec, + ), + ), + ) + + +def compile_service( + *, + resource_name: str, + workspace: str, + name: str, + engine: str, + port: int = 8000, + namespace: Optional[str] = None, +) -> k8s_client.V1Service: + """Compile the ClusterIP Service exposing the server port for IGW routing.""" + return k8s_client.V1Service( + metadata=k8s_client.V1ObjectMeta( + name=resource_name, + namespace=namespace, + labels=common_labels(workspace, name, engine), + ), + spec=k8s_client.V1ServiceSpec( + type="ClusterIP", + selector={"app": resource_name}, + ports=[ + k8s_client.V1ServicePort( + name=SERVER_PORT_NAME, + port=port, + target_port=SERVER_PORT_NAME, + protocol="TCP", + ), + ], + ), + ) diff --git a/services/core/models/src/nmp/core/models/controllers/backends/none_backend.py b/services/core/models/src/nmp/core/models/controllers/backends/none_backend.py index d77cd319eb..9f5e42560b 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/none_backend.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/none_backend.py @@ -3,12 +3,8 @@ """None service backend.""" -from typing import Optional - -from nemo_platform.types.inference.model_deployment import ModelDeployment -from nemo_platform.types.inference.model_deployment_config import ModelDeploymentConfig -from nemo_platform.types.models.model_entity import ModelEntity from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend +from nmp.core.models.controllers.context import ModelContext class NoneServiceBackend(ServiceBackend): @@ -22,19 +18,15 @@ def shutdown(self) -> None: """Shutdown None service backend.""" ... - async def create_model_deployment( - self, deployment: ModelDeployment, config: ModelDeploymentConfig, model_entity: Optional[ModelEntity] = None - ) -> DeploymentStatusUpdate: + async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Create a new model deployment.""" raise NotImplementedError("NoneServiceBackend does not support deployments") - async def update_model_deployment( - self, deployment: ModelDeployment, config: ModelDeploymentConfig, model_entity: Optional[ModelEntity] = None - ) -> DeploymentStatusUpdate: + async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Update a model deployment.""" raise NotImplementedError("NoneServiceBackend does not support deployments") - async def get_model_deployment_status(self, deployment: ModelDeployment) -> DeploymentStatusUpdate: + async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Get the status of a model deployment.""" return DeploymentStatusUpdate( status="UNKNOWN", diff --git a/services/core/models/src/nmp/core/models/controllers/backends/docker/vllm_compiler.py b/services/core/models/src/nmp/core/models/controllers/backends/vllm_compiler.py similarity index 93% rename from services/core/models/src/nmp/core/models/controllers/backends/docker/vllm_compiler.py rename to services/core/models/src/nmp/core/models/controllers/backends/vllm_compiler.py index ad86b6bc79..26b42acaea 100644 --- a/services/core/models/src/nmp/core/models/controllers/backends/docker/vllm_compiler.py +++ b/services/core/models/src/nmp/core/models/controllers/backends/vllm_compiler.py @@ -1,13 +1,16 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""vLLM compiler for the docker service backend. +"""Backend-agnostic vLLM compiler. Turns a ``ModelDeploymentConfig`` whose ``engine`` is ``vllm`` into the image, -``vllm serve`` argument vector, and environment variables for a ``docker run`` -container. The shared creation pipeline (GPU/port/volume allocation, the weight -puller, container start) is reused from ``DockerDeploymentCreationReconciler``; -this module only produces the vLLM-specific resource shape. +``vllm serve`` argument vector, and environment variables for a vLLM server. + +These functions take a :class:`DeploymentConfigView` and a ``ModelEntity`` and +return plain data (arg vectors, env dicts, image tuples, TP sizing) -- they are +NOT specific to any service backend. The docker backend renders the result into +a ``docker run`` container; the k8s backend renders it into native Kubernetes +objects. Keep this module free of backend-specific imports so both can reuse it. """ from logging import getLogger diff --git a/services/core/models/src/nmp/core/models/controllers/deployment_reconciler.py b/services/core/models/src/nmp/core/models/controllers/deployment_reconciler.py index e1d0df48d8..6c2dad5c63 100644 --- a/services/core/models/src/nmp/core/models/controllers/deployment_reconciler.py +++ b/services/core/models/src/nmp/core/models/controllers/deployment_reconciler.py @@ -186,20 +186,20 @@ async def reconcile_deployments(self, deployment_contexts: list[ModelContext]) - match deployment.status: case "CREATED": - # Use pre-fetched config and entity from context - config = ctx.model_deployment_config - model_entity = ctx.model_entity - - # Lambda needed to bind the config and model_entity arguments + # Lambda needed to bind ctx (the reconcile context bundles + # the deployment, config, and model entity). await self._reconcile_individual_deployment( deployment, - lambda dep: backend.create_model_deployment(dep, config, model_entity), + lambda _dep, _ctx=ctx: backend.create_model_deployment(_ctx), "create", existing_provider=ctx.model_provider, ) case "PENDING" | "READY" | "UNKNOWN": - # Check status and handle drift/backend issues - status_update = await backend.get_model_deployment_status(deployment) + # Check status and handle drift/backend issues. The ctx + # carries the config + entity so backends that advance + # creation in the status path (k8s vLLM) can compile the + # serving objects. + status_update = await backend.get_model_deployment_status(ctx) if status_update.status == "LOST": # Drift detected - attempt recovery @@ -213,11 +213,14 @@ async def reconcile_deployments(self, deployment_contexts: list[ModelContext]) - # Clear recovery state - deployment is healthy or in terminal state self._drift_recovery_cache.remove(model_deployment_id) - # Process the status update + # Process the status update. ``status_update`` is already + # fetched above, so ``_reconcile_individual_deployment`` + # won't invoke this callable; it's passed only for the + # generic signature (bind ctx for type consistency). action = "check status of" if deployment.status == "PENDING" else "monitor" await self._reconcile_individual_deployment( deployment, - backend.get_model_deployment_status, + lambda _dep, _ctx=ctx: backend.get_model_deployment_status(_ctx), action, existing_provider=ctx.model_provider, status_update=status_update, @@ -507,11 +510,7 @@ async def _handle_drift_recovery( try: # Call create_model_deployment to recreate resources - status_update = await backend.create_model_deployment( - deployment, - ctx.model_deployment_config, - ctx.model_entity, - ) + status_update = await backend.create_model_deployment(ctx) # Build recovery message recovery_message = ( diff --git a/services/core/models/tests/integration/conftest.py b/services/core/models/tests/integration/conftest.py index 0328a8158d..9a0b06b4ff 100644 --- a/services/core/models/tests/integration/conftest.py +++ b/services/core/models/tests/integration/conftest.py @@ -19,6 +19,7 @@ from nmp.core.files.app.backends.huggingface import HuggingfaceStorageImpl from nmp.core.models.controllers.backends.backends import DeploymentStatusUpdate, ServiceBackend from nmp.core.models.controllers.backends.registry import BackendRegistry +from nmp.core.models.controllers.context import ModelContext from nmp.core.models.controllers.models_controller import ModelsController from nmp.core.models.service import ModelsService from nmp.core.secrets.config import SecretsServiceConfig @@ -153,32 +154,27 @@ def shutdown(self) -> None: """No-op shutdown for mock backend.""" pass - async def create_model_deployment( - self, - deployment: ModelDeployment, - config: ModelDeploymentConfig, - model_entity: Optional[ModelEntity] = None, - ) -> DeploymentStatusUpdate: + async def create_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Record call and return configured response.""" - self.create_calls.append((deployment, config, model_entity)) + self.create_calls.append((ctx.model_deployment, ctx.model_deployment_config, ctx.model_entity)) return self.create_response - async def update_model_deployment( - self, - deployment: ModelDeployment, - config: ModelDeploymentConfig, - model_entity: Optional[ModelEntity] = None, - ) -> DeploymentStatusUpdate: + async def update_model_deployment(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Record call and return configured response.""" - self.update_calls.append((deployment, config, model_entity)) + self.update_calls.append((ctx.model_deployment, ctx.model_deployment_config, ctx.model_entity)) return self.create_response # Update returns same as create - async def get_model_deployment_status(self, deployment: ModelDeployment) -> DeploymentStatusUpdate: + async def get_model_deployment_status(self, ctx: ModelContext) -> DeploymentStatusUpdate: """Record call and return configured response. + Takes the reconcile ``ctx`` (bundling deployment + config + entity) like the + real ``ServiceBackend``; records the deployment so existing assertions that + inspect ``status_calls`` by ``.name`` keep working. + Uses per-deployment responses from status_responses dict if available, otherwise falls back to default_status_response. """ + deployment = ctx.model_deployment self.status_calls.append(deployment) return self.status_responses.get(deployment.name, self.default_status_response) diff --git a/services/core/models/tests/integration/test_models.py b/services/core/models/tests/integration/test_models.py index bd9daedc0e..bcd458fe8d 100644 --- a/services/core/models/tests/integration/test_models.py +++ b/services/core/models/tests/integration/test_models.py @@ -1566,9 +1566,6 @@ def test_backend_config_key_k8s_works_end_to_end(): patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_config.load_kube_config"), patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_client.ApiClient"), patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.DynamicClient"), - patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.backend.K8sNimOperatorServiceBackend._validate_nim_operator_crds" - ), ): registry = BackendRegistry.from_config( nmp_sdk=AsyncMock(), diff --git a/services/core/models/tests/unit/controllers/backends/docker/test_vllm_compiler.py b/services/core/models/tests/unit/controllers/backends/test_vllm_compiler.py similarity index 98% rename from services/core/models/tests/unit/controllers/backends/docker/test_vllm_compiler.py rename to services/core/models/tests/unit/controllers/backends/test_vllm_compiler.py index 5528bc95d7..48d8eb55e7 100644 --- a/services/core/models/tests/unit/controllers/backends/docker/test_vllm_compiler.py +++ b/services/core/models/tests/unit/controllers/backends/test_vllm_compiler.py @@ -1,12 +1,12 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Unit tests for the docker backend vLLM compiler.""" +"""Unit tests for the backend-agnostic vLLM compiler.""" from types import SimpleNamespace +from nmp.core.models.controllers.backends import vllm_compiler from nmp.core.models.controllers.backends.common import DeploymentConfigView -from nmp.core.models.controllers.backends.docker import vllm_compiler def _view(**kwargs) -> DeploymentConfigView: diff --git a/services/core/models/tests/unit/controllers/backends/test_vllm_k8s_compiler.py b/services/core/models/tests/unit/controllers/backends/test_vllm_k8s_compiler.py new file mode 100644 index 0000000000..75a5488031 --- /dev/null +++ b/services/core/models/tests/unit/controllers/backends/test_vllm_k8s_compiler.py @@ -0,0 +1,305 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit tests for the engine-agnostic k8s object compiler (vLLM raw-object path).""" + +from nmp.core.models.app.constants import MODEL_MANAGED_BY_LABEL, MODEL_MANAGED_BY_MODELS_CONTROLLER +from nmp.core.models.controllers.backends.k8s_nim_operator import vllm_k8s_compiler as c + +# --------------------------------------------------------------------------- +# Naming + labels +# --------------------------------------------------------------------------- + + +def test_resource_name_suffixes(): + assert c.pvc_name("md-default-qwen") == "md-default-qwen-pvc" + assert c.pull_job_name("md-default-qwen") == "md-default-qwen-pull" + + +def test_common_labels(): + labels = c.common_labels("default", "qwen", "vllm") + assert labels[MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER + assert labels[c.DEPLOYMENT_WORKSPACE_LABEL] == "default" + assert labels[c.DEPLOYMENT_NAME_LABEL] == "qwen" + assert labels["nmp.nvidia.com/engine"] == "vllm" + + +# --------------------------------------------------------------------------- +# PVC +# --------------------------------------------------------------------------- + + +def test_compile_pvc_basic(): + pvc = c.compile_pvc( + resource_name="md-default-qwen", + workspace="default", + name="qwen", + engine="vllm", + disk_size="50Gi", + namespace="nemo", + ) + assert pvc.metadata.name == "md-default-qwen-pvc" + assert pvc.metadata.namespace == "nemo" + assert pvc.spec.access_modes == ["ReadWriteOnce"] + assert pvc.spec.resources.requests["storage"] == "50Gi" + assert pvc.spec.storage_class_name is None + + +def test_compile_pvc_storage_class_and_model_source(): + pvc = c.compile_pvc( + resource_name="md-default-qwen", + workspace="default", + name="qwen", + engine="vllm", + disk_size="100Gi", + storage_class="fast-ssd", + model_source="default/qwen@main", + ) + assert pvc.spec.storage_class_name == "fast-ssd" + # model source is an annotation (its value contains '/' and '@', invalid for labels). + assert pvc.metadata.annotations[c.MODEL_SOURCE_ANNOTATION] == "default/qwen@main" + + +def test_compile_pvc_custom_access_modes(): + pvc = c.compile_pvc( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + disk_size="10Gi", + access_modes=["ReadWriteMany"], + ) + assert pvc.spec.access_modes == ["ReadWriteMany"] + + +# --------------------------------------------------------------------------- +# Puller Job +# --------------------------------------------------------------------------- + + +def test_compile_puller_job_basic(): + job = c.compile_puller_job( + resource_name="md-default-qwen", + workspace="default", + name="qwen", + engine="vllm", + image="hf-cli:25.10", + container_args=["download", "default/qwen", "--local-dir", "/model-store"], + env={"HF_ENDPOINT": "http://files/apis/files/v2/hf", "HF_TOKEN": "service:models"}, + gpu=2, + namespace="nemo", + service_account_name="nemo-models-sa", + image_pull_secret="nvcrimagepullsecret", + model_source="default/qwen@main", + ) + assert job.metadata.name == "md-default-qwen-pull" + assert job.spec.backoff_limit == c.DEFAULT_BACKOFF_LIMIT + assert job.spec.ttl_seconds_after_finished == c.DEFAULT_TTL_SECONDS_AFTER_FINISHED + + pod = job.spec.template.spec + assert pod.restart_policy == "Never" + assert pod.service_account_name == "nemo-models-sa" + assert pod.image_pull_secrets[0].name == "nvcrimagepullsecret" + + ctr = pod.containers[0] + assert ctr.args == ["download", "default/qwen", "--local-dir", "/model-store"] + # Entrypoint overridden to the HF CLI (nmp-api's image entrypoint is `nemo + # services run`); args run as `hf download ...`. + assert ctr.command == ["hf"] + env = {e.name: e.value for e in ctr.env} + assert env["HF_ENDPOINT"] == "http://files/apis/files/v2/hf" + assert env["HF_TOKEN"] == "service:models" + # GPU request pins the puller into GPU topology for PVC binding. + assert ctr.resources.requests["nvidia.com/gpu"] == "2" + assert ctr.resources.limits["nvidia.com/gpu"] == "2" + assert ctr.volume_mounts[0].mount_path == "/model-store" + # Job annotation carries the model source for the re-pull policy. + assert job.metadata.annotations[c.MODEL_SOURCE_ANNOTATION] == "default/qwen@main" + + +def test_compile_puller_job_cpu_only_no_gpu_request(): + job = c.compile_puller_job( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + image="hf-cli", + container_args=["download", "w/n", "--local-dir", "/model-store"], + gpu=0, + ) + assert job.spec.template.spec.containers[0].resources is None + + +def test_compile_puller_job_no_image_pull_secret(): + job = c.compile_puller_job( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + image="hf-cli", + container_args=["download"], + ) + assert job.spec.template.spec.image_pull_secrets is None + + +# --------------------------------------------------------------------------- +# Deployment +# --------------------------------------------------------------------------- + + +def test_compile_deployment_basic(): + dep = c.compile_deployment( + resource_name="md-default-qwen", + workspace="default", + name="qwen", + engine="vllm", + image="vllm/vllm-openai:v0.22.1", + args=["/model-store", "--served-model-name", "default/qwen"], + health_path="/health", + gpu=2, + namespace="nemo", + service_account_name="nemo-models-sa", + ) + assert dep.metadata.name == "md-default-qwen" + assert dep.spec.replicas == 1 + assert dep.spec.selector.match_labels == {"app": "md-default-qwen"} + + pod = dep.spec.template.spec + assert pod.service_account_name == "nemo-models-sa" + ctr = pod.containers[0] + # command unset -> image entrypoint (vllm serve) runs; args appended. + assert ctr.command is None + assert ctr.args == ["/model-store", "--served-model-name", "default/qwen"] + assert ctr.ports[0].container_port == 8000 + assert ctr.resources.limits["nvidia.com/gpu"] == "2" + assert ctr.startup_probe.http_get.path == "/health" + assert ctr.readiness_probe.http_get.path == "/health" + + # PVC mounted read-only at /model-store; scratch + dshm present. + mounts = {m.name: m for m in ctr.volume_mounts} + assert mounts["model-store"].mount_path == "/model-store" + assert mounts["model-store"].read_only is True + assert mounts["scratch"].mount_path == "/scratch" + assert mounts["dshm"].mount_path == "/dev/shm" + + vols = {v.name: v for v in pod.volumes} + assert vols["model-store"].persistent_volume_claim.claim_name == "md-default-qwen-pvc" + assert vols["dshm"].empty_dir.medium == "Memory" + + +def test_compile_deployment_cpu_only_no_gpu(): + dep = c.compile_deployment( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + image="img", + args=["/model-store"], + health_path="/health", + gpu=0, + ) + assert dep.spec.template.spec.containers[0].resources is None + + +def test_compile_deployment_startup_grace_to_failure_threshold(): + dep = c.compile_deployment( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + image="img", + args=[], + health_path="/health", + startup_grace_seconds=600, + ) + # ceil(600 / 10) == 60 + assert dep.spec.template.spec.containers[0].startup_probe.failure_threshold == 60 + + +def test_compile_deployment_shared_memory_size_limit(): + dep = c.compile_deployment( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + image="img", + args=[], + health_path="/health", + shared_memory_size_limit="8Gi", + ) + vols = {v.name: v for v in dep.spec.template.spec.volumes} + assert vols["dshm"].empty_dir.size_limit == "8Gi" + + +def test_compile_deployment_security_context_set_when_uid_gid_given(): + """When uid/gid are provided, the server pod gets that securityContext.""" + dep = c.compile_deployment( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + image="img", + args=[], + health_path="/health", + user_id=2000, + group_id=0, + ) + sc = dep.spec.template.spec.security_context + assert sc.run_as_user == 2000 + assert sc.run_as_group == 0 + assert sc.fs_group == 0 + + +def test_compile_deployment_no_security_context_when_uid_gid_unset(): + """No uid/gid -> no forced securityContext (runs as the image's default user).""" + dep = c.compile_deployment( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + image="img", + args=[], + health_path="/health", + ) + assert dep.spec.template.spec.security_context is None + + +def test_compile_deployment_sidecars_and_init_containers(): + from kubernetes import client as k8s_client + + sidecar = k8s_client.V1Container(name="lora-sidecar", image="nmp-api") + init = k8s_client.V1Container(name="lora-cache-init", image="busybox") + dep = c.compile_deployment( + resource_name="r", + workspace="w", + name="n", + engine="vllm", + image="img", + args=[], + health_path="/health", + init_containers=[init], + sidecar_containers=[sidecar], + ) + pod = dep.spec.template.spec + assert pod.init_containers[0].name == "lora-cache-init" + assert [ctr.name for ctr in pod.containers] == ["r-ctr", "lora-sidecar"] + + +# --------------------------------------------------------------------------- +# Service +# --------------------------------------------------------------------------- + + +def test_compile_service_basic(): + svc = c.compile_service( + resource_name="md-default-qwen", + workspace="default", + name="qwen", + engine="vllm", + namespace="nemo", + ) + assert svc.spec.type == "ClusterIP" + assert svc.spec.selector == {"app": "md-default-qwen"} + assert svc.spec.ports[0].port == 8000 + assert svc.spec.ports[0].target_port == c.SERVER_PORT_NAME + assert svc.metadata.labels[MODEL_MANAGED_BY_LABEL] == MODEL_MANAGED_BY_MODELS_CONTROLLER diff --git a/services/core/models/tests/unit/controllers/test_backend_config_fields.py b/services/core/models/tests/unit/controllers/test_backend_config_fields.py index ad03c4d383..da07d7d967 100644 --- a/services/core/models/tests/unit/controllers/test_backend_config_fields.py +++ b/services/core/models/tests/unit/controllers/test_backend_config_fields.py @@ -595,3 +595,60 @@ def test_default_annotations_applied_to_nimservice_metadata_and_spec(sample_depl assert nimservice.metadata["annotations"] == {"prometheus.io/scrape": "true", "custom/key": "value"} assert nimservice.spec.annotations == {"prometheus.io/scrape": "true", "custom/key": "value"} + + +# --------------------------------------------------------------------------- +# vLLM-on-k8s config fields (raw-object emission path) +# --------------------------------------------------------------------------- + + +def test_default_vllm_image_default_value(): + """default_vllm_image / _tag fall back to the upstream vLLM image.""" + backend_config = K8sNimOperatorConfig() + assert backend_config.default_vllm_image == "vllm/vllm-openai" + assert backend_config.default_vllm_image_tag == "v0.22.1" + + +def test_default_vllm_image_override(): + """default_vllm_image / _tag can be repointed at a mirror.""" + backend_config = K8sNimOperatorConfig( + default_vllm_image="my-registry/vllm-openai", + default_vllm_image_tag="v0.99.0", + ) + assert backend_config.default_vllm_image == "my-registry/vllm-openai" + assert backend_config.default_vllm_image_tag == "v0.99.0" + + +def test_service_account_name_defaults_to_none(): + """service_account_name defaults to None (namespace default ServiceAccount).""" + assert K8sNimOperatorConfig().service_account_name is None + + +def test_service_account_name_override(): + """service_account_name can be set to a shared models ServiceAccount.""" + backend_config = K8sNimOperatorConfig(service_account_name="nemo-models-sa") + assert backend_config.service_account_name == "nemo-models-sa" + + +def test_default_shared_memory_size_limit_defaults_to_none(): + """default_shared_memory_size_limit defaults to None (node default /dev/shm).""" + assert K8sNimOperatorConfig().default_shared_memory_size_limit is None + + +def test_default_shared_memory_size_limit_override(): + """default_shared_memory_size_limit can be set for vLLM tensor-parallel NCCL.""" + backend_config = K8sNimOperatorConfig(default_shared_memory_size_limit="8Gi") + assert backend_config.default_shared_memory_size_limit == "8Gi" + + +def test_default_vllm_uid_gid_match_image_user(): + """vLLM uid/gid default to the upstream image's 'vllm' user (2000) / root group (0).""" + backend_config = K8sNimOperatorConfig() + assert backend_config.default_vllm_user_id == 2000 + assert backend_config.default_vllm_group_id == 0 + + +def test_default_vllm_uid_gid_override(): + backend_config = K8sNimOperatorConfig(default_vllm_user_id=1234, default_vllm_group_id=5678) + assert backend_config.default_vllm_user_id == 1234 + assert backend_config.default_vllm_group_id == 5678 diff --git a/services/core/models/tests/unit/controllers/test_backend_registry.py b/services/core/models/tests/unit/controllers/test_backend_registry.py index 8e99ed1f14..28fad677ba 100644 --- a/services/core/models/tests/unit/controllers/test_backend_registry.py +++ b/services/core/models/tests/unit/controllers/test_backend_registry.py @@ -39,9 +39,6 @@ def mock_k8s_config(): patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_config.load_kube_config"), patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.k8s_client.ApiClient"), patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.DynamicClient"), - patch( - "nmp.core.models.controllers.backends.k8s_nim_operator.backend.K8sNimOperatorServiceBackend._validate_nim_operator_crds" - ), ): yield diff --git a/services/core/models/tests/unit/controllers/test_deployment_reconciler.py b/services/core/models/tests/unit/controllers/test_deployment_reconciler.py index 12fe2bffa8..4b73d60a75 100644 --- a/services/core/models/tests/unit/controllers/test_deployment_reconciler.py +++ b/services/core/models/tests/unit/controllers/test_deployment_reconciler.py @@ -272,12 +272,13 @@ async def test_reconcile_deployments_with_created_status(reconciler, mock_backen # Process deployments (now passing contexts with pre-fetched data) await reconciler.reconcile_deployments([created_context, pending_context]) - # Verify backend.create was called for CREATED deployment with the config and model_entity - # Config is already in context so no retrieve call happens during reconciliation - mock_backend.create_model_deployment.assert_called_once_with(created_deployment, mock_deployment_config, None) + # Verify backend.create was called for CREATED deployment with the full context + # (the context bundles deployment + config + entity; already pre-fetched, so no + # retrieve happens during reconciliation). + mock_backend.create_model_deployment.assert_called_once_with(created_context) - # Verify backend.get_status was called for PENDING deployment - mock_backend.get_model_deployment_status.assert_called_once_with(pending_deployment) + # Verify backend.get_status was called for PENDING deployment with the context. + mock_backend.get_model_deployment_status.assert_called_once_with(pending_context) # Verify SDK update was called twice (once for each deployment) assert reconciler._models_sdk.inference.deployments.update_status.call_count == 2 @@ -1133,8 +1134,8 @@ async def test_lost_status_triggers_drift_recovery(reconciler, mock_backend_regi # Process deployment await reconciler.reconcile_deployments([ctx]) - # Verify backend.create_model_deployment was called for recovery - mock_backend.create_model_deployment.assert_called_once_with(deployment, mock_deployment_config, None) + # Verify backend.create_model_deployment was called for recovery with the context + mock_backend.create_model_deployment.assert_called_once_with(ctx) # Verify status was updated to PENDING with recovery message reconciler._models_sdk.inference.deployments.update_status.assert_called_once() diff --git a/services/core/models/tests/unit/controllers/test_docker_backend.py b/services/core/models/tests/unit/controllers/test_docker_backend.py index 52c893c244..63298357c1 100644 --- a/services/core/models/tests/unit/controllers/test_docker_backend.py +++ b/services/core/models/tests/unit/controllers/test_docker_backend.py @@ -18,6 +18,7 @@ CreationStage, _compute_multi_gpu_shm_size, ) +from nmp.core.models.controllers.context import ModelContext from nmp.core.models.schemas import ( ContainerExecutorConfig, Engine, @@ -86,7 +87,7 @@ async def drive_creation_to_completion(backend: DockerServiceBackend, deployment return last_status if last_status is not None: return last_status - return await backend.get_model_deployment_status(deployment) + return await backend.get_model_deployment_status(ModelContext(model_deployment=deployment)) @pytest.fixture @@ -350,7 +351,9 @@ async def test_docker_backend_create_model_deployment( mock_docker_client.containers.list.return_value = [] # create_model_deployment now starts the image pull and returns PENDING immediately - initial_status = await docker_backend.create_model_deployment(sample_deployment, sample_config) + initial_status = await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) assert initial_status.status == "PENDING" assert "pulling container image" in initial_status.status_message.lower() @@ -429,7 +432,11 @@ async def _drive_vllm_with_puller(docker_backend, sample_deployment, mock_docker mock_docker_client.containers.create.return_value = mock_vllm_container mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, config, _vllm_model_entity()) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, model_deployment_config=config, model_entity=_vllm_model_entity() + ) + ) def get_container_side_effect(name): if "puller" in name: @@ -591,7 +598,11 @@ async def test_docker_backend_create_sft_model_success( mock_docker_client.containers.list.return_value = [] # create_model_deployment starts the pipeline; drive it to completion - await docker_backend.create_model_deployment(sample_deployment, sample_config, model_entity) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity + ) + ) # Mock puller container for get/reload polling mock_puller_container.status = "exited" @@ -670,7 +681,11 @@ async def test_docker_backend_create_sft_model_puller_fails( # Mock containers.list to return empty (no ports in use) mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, sample_config, model_entity) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity + ) + ) # Mock get for puller container polling def get_container_side_effect(name): @@ -705,7 +720,7 @@ async def test_docker_backend_get_model_deployment_status_running( mock_docker_client.containers.get.return_value = mock_container mock_docker_client.containers.get.side_effect = None - status_update = await docker_backend.get_model_deployment_status(sample_deployment) + status_update = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) # Verify status update assert status_update is not None @@ -721,7 +736,7 @@ async def test_docker_backend_get_model_deployment_status_not_found( """Test getting status when container is not found.""" mock_docker_client.containers.get.side_effect = NotFound("Container not found") - status_update = await docker_backend.get_model_deployment_status(sample_deployment) + status_update = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) # Verify status update assert status_update is not None @@ -1006,7 +1021,9 @@ async def test_docker_backend_create_with_port_forwarding( # Mock containers.list to return empty (no ports in use) mock_docker_client.containers.list.return_value = [] - await docker_backend_with_dind_mode.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_dind_mode.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status_update = await drive_creation_to_completion(docker_backend_with_dind_mode, sample_deployment) # Verify status update @@ -1040,7 +1057,9 @@ async def test_docker_backend_get_status_with_port_forwarding( mock_docker_client.containers.get.side_effect = None mock_docker_client.containers.get.return_value = mock_container - status_update = await docker_backend_with_dind_mode.get_model_deployment_status(sample_deployment) + status_update = await docker_backend_with_dind_mode.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment) + ) # Verify status and URL uses the port from container bindings assert status_update.status == "READY" @@ -1100,7 +1119,9 @@ async def test_docker_backend_port_exhaustion_error( mock_docker_client.containers.list.return_value = mock_containers - await docker_backend_with_dind_mode.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_dind_mode.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status_update = await drive_creation_to_completion(docker_backend_with_dind_mode, sample_deployment) # Should return ERROR status (port exhaustion happens during container creation stage) @@ -1138,7 +1159,9 @@ def mock_containers_list(**kwargs): deployment.name = f"model-{i}" deployments.append(deployment) - await docker_backend_with_dind_mode.create_model_deployment(deployment, sample_config) + await docker_backend_with_dind_mode.create_model_deployment( + ModelContext(model_deployment=deployment, model_deployment_config=sample_config) + ) status = await drive_creation_to_completion(docker_backend_with_dind_mode, deployment) # Extract port from host_url (format: http://docker:PORT) @@ -1272,7 +1295,13 @@ async def test_multi_llm_sft_model_now_runs_puller_old_test_updated( _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - await docker_backend.create_model_deployment(sample_deployment, multi_llm_config, sft_model_entity) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=multi_llm_config, + model_entity=sft_model_entity, + ) + ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) assert status_update is not None @@ -1316,7 +1345,13 @@ async def test_model_specific_nim_sft_model_runs_puller( _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - await docker_backend.create_model_deployment(sample_deployment, model_specific_nim_config, sft_model_entity) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=model_specific_nim_config, + model_entity=sft_model_entity, + ) + ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) assert status_update is not None @@ -1354,7 +1389,11 @@ async def test_explicit_multi_llm_image_now_runs_puller( _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) await docker_backend.create_model_deployment( - sample_deployment, explicit_multi_llm_config, sft_model_entity_with_artifact + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=explicit_multi_llm_config, + model_entity=sft_model_entity_with_artifact, + ) ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) @@ -1377,7 +1416,9 @@ async def test_multi_llm_non_sft_model_fails_without_supported_weights_type( mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, multi_llm_config, model_entity=None) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=multi_llm_config, model_entity=None) + ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) assert status_update is not None @@ -1466,7 +1507,13 @@ async def test_multi_llm_now_runs_puller( _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - await docker_backend.create_model_deployment(sample_deployment, multi_llm_config, sft_model_entity_with_artifact) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=multi_llm_config, + model_entity=sft_model_entity_with_artifact, + ) + ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) assert status_update is not None @@ -1930,7 +1977,9 @@ async def test_docker_backend_create_with_dond_mode_uses_container_name_url( # Mock containers.list to return empty (no ports in use) mock_docker_client.containers.list.return_value = [] - await docker_backend_with_dond_mode.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_dond_mode.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status_update = await drive_creation_to_completion(docker_backend_with_dond_mode, sample_deployment) # Verify status update @@ -1962,7 +2011,9 @@ async def test_docker_backend_get_status_with_dond_mode_uses_container_name_url( mock_docker_client.containers.get.side_effect = None mock_docker_client.containers.get.return_value = mock_container - status_update = await docker_backend_with_dond_mode.get_model_deployment_status(sample_deployment) + status_update = await docker_backend_with_dond_mode.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment) + ) # Verify status and URL uses container name (DonD mode ignores port bindings) assert status_update.status == "READY" @@ -1985,7 +2036,9 @@ async def test_docker_backend_dond_mode_container_joins_network( mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - await docker_backend_with_dond_mode.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_dond_mode.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend_with_dond_mode, sample_deployment) # Verify every container was created with network attached @@ -2089,7 +2142,9 @@ async def test_docker_backend_allocates_gpu_from_pool( # Initially all GPUs available assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 - await docker_backend_with_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) # One GPU should be allocated @@ -2113,7 +2168,9 @@ async def test_docker_backend_fails_without_gpu_pool( mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - await docker_backend_without_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_without_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status_update = await drive_creation_to_completion(docker_backend_without_gpu_pool, sample_deployment) assert status_update.status == "ERROR" @@ -2138,7 +2195,9 @@ async def test_docker_backend_releases_gpu_on_delete( mock_docker_client.containers.list.return_value = [] # Create deployment and drive to completion - allocates GPU - await docker_backend_with_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 @@ -2177,7 +2236,9 @@ async def test_docker_backend_gpu_allocation_failure( assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 0 # Try to create deployment - GPU allocation failure happens during container creation stage - await docker_backend_with_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status_update = await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) assert status_update.status == "ERROR" @@ -2205,7 +2266,9 @@ async def test_docker_backend_releases_gpu_on_port_allocation_failure( assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 # Try to create deployment - port allocation failure happens during container creation stage - await docker_backend_with_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status_update = await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) assert status_update.status == "ERROR" @@ -2237,7 +2300,9 @@ async def test_docker_backend_releases_gpu_on_container_creation_failure( assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 4 # Try to create deployment - container creation failure happens during CREATING_CONTAINER stage - await docker_backend_with_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status_update = await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) assert status_update.status == "ERROR" @@ -2294,7 +2359,9 @@ async def test_docker_backend_multi_gpu_allocation(mock_nmp_sdk, mock_docker_cli mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - await backend.create_model_deployment(sample_deployment, multi_gpu_config) + await backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=multi_gpu_config) + ) await drive_creation_to_completion(backend, sample_deployment) # Should have allocated 2 GPUs @@ -2339,7 +2406,9 @@ async def test_docker_backend_releases_gpu_on_status_check_terminated( mock_docker_client.containers.list.return_value = [] # Create deployment and drive to completion - allocates GPU - await docker_backend_with_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 @@ -2350,7 +2419,9 @@ async def test_docker_backend_releases_gpu_on_status_check_terminated( mock_docker_client.containers.get.return_value = mock_container # Check status - should release GPU when container is terminated - status_update = await docker_backend_with_gpu_pool.get_model_deployment_status(sample_deployment) + status_update = await docker_backend_with_gpu_pool.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment) + ) # Verify ERROR status returned assert status_update.status == "ERROR" @@ -2378,7 +2449,9 @@ async def test_docker_backend_releases_gpu_on_status_check_lost( mock_docker_client.containers.list.return_value = [] # Create deployment and drive to completion - allocates GPU - await docker_backend_with_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 @@ -2386,7 +2459,9 @@ async def test_docker_backend_releases_gpu_on_status_check_lost( mock_docker_client.containers.get.side_effect = NotFound("Container not found") # Check status - should release GPU when container is missing - status_update = await docker_backend_with_gpu_pool.get_model_deployment_status(sample_deployment) + status_update = await docker_backend_with_gpu_pool.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment) + ) # Verify LOST status returned assert status_update.status == "LOST" @@ -2418,7 +2493,9 @@ async def test_docker_backend_status_check_does_not_release_gpu_when_running( docker_backend_with_gpu_pool._backend_config.models_docker_networking_mode = "dond" # Create deployment and drive to completion - allocates GPU - await docker_backend_with_gpu_pool.create_model_deployment(sample_deployment, sample_config) + await docker_backend_with_gpu_pool.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend_with_gpu_pool, sample_deployment) assert docker_backend_with_gpu_pool._gpu_pool.get_available_count() == 3 @@ -2429,7 +2506,9 @@ async def test_docker_backend_status_check_does_not_release_gpu_when_running( mock_docker_client.containers.get.return_value = mock_container # Check status - should NOT release GPU when container is running - status_update = await docker_backend_with_gpu_pool.get_model_deployment_status(sample_deployment) + status_update = await docker_backend_with_gpu_pool.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment) + ) # Verify READY status returned assert status_update.status == "READY" @@ -2510,7 +2589,11 @@ async def test_multi_llm_files_service_deployment_succeeds( _setup_puller_mock_for_polling(mock_docker_client, deployment, exit_code=0) # No model entity (matches the bug scenario) - await docker_backend.create_model_deployment(deployment, multi_llm_config_with_model_name, model_entity=None) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=deployment, model_deployment_config=multi_llm_config_with_model_name, model_entity=None + ) + ) status_update = await drive_creation_to_completion(docker_backend, deployment) # Should succeed with PENDING status @@ -2662,7 +2745,9 @@ async def test_multi_llm_huggingface_deployment_succeeds_with_hf_token( _setup_puller_mock_for_polling(mock_docker_client, deployment, exit_code=0) - await docker_backend.create_model_deployment(deployment, config, model_entity=None) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=deployment, model_deployment_config=config, model_entity=None) + ) status_update = await drive_creation_to_completion(docker_backend, deployment) assert status_update.status == "PENDING", ( @@ -2717,7 +2802,9 @@ async def test_create_releases_stale_gpu_allocation_single( mock_docker_client.containers.list.return_value = [] # Create deployment and drive to completion (should release stale allocation first, then reallocate) - await docker_backend.create_model_deployment(sample_deployment, sample_config, None) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=None) + ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) assert status_update.status == "PENDING" @@ -2761,7 +2848,9 @@ async def test_create_releases_stale_gpu_allocation_multi(docker_backend, sample mock_docker_client.containers.create.return_value = mock_container mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, config, None) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) + ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) assert status_update.status == "PENDING" @@ -2789,7 +2878,9 @@ async def test_create_without_stale_allocation_succeeds( mock_docker_client.containers.create.return_value = mock_container mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, sample_config, None) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=None) + ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) assert status_update.status == "PENDING" @@ -3535,7 +3626,9 @@ async def test_create_deployment_with_tool_call_plugin_from_entity( mock_docker_client.containers.create.return_value = mock_nim_container mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, config, model_entity) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=model_entity) + ) status_update = await drive_creation_to_completion(docker_backend, sample_deployment) assert status_update.status == "PENDING" @@ -3608,7 +3701,7 @@ async def test_running_not_healthy_within_timeout_returns_pending( with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "PENDING" assert "still initializing" in status.status_message @@ -3625,7 +3718,7 @@ async def test_running_not_healthy_exceeds_timeout_returns_error( with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "ERROR" assert "timed out" in status.status_message @@ -3641,7 +3734,7 @@ async def test_created_state_within_timeout_returns_pending( """A container in 'created' state should return PENDING with timing info.""" make_mock_container(status="created") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "PENDING" assert "starting up" in status.status_message @@ -3654,7 +3747,7 @@ async def test_restarting_state_within_timeout_returns_pending_with_restart_coun """A container in 'restarting' state should include restart count.""" make_mock_container(status="restarting", restart_count=3) - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "PENDING" assert "restart count: 3" in status.status_message @@ -3669,7 +3762,7 @@ async def test_restarting_state_exceeds_timeout_returns_error( docker_backend._backend_config.pending_timeout_seconds = 3600 sample_deployment.created_at = datetime.now(timezone.utc) - timedelta(hours=2) - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "ERROR" assert "timed out" in status.status_message @@ -3682,7 +3775,7 @@ async def test_ready_returns_ready_status(self, docker_backend, sample_deploymen """A healthy container should return READY.""" make_mock_container(status="running") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "READY" @@ -3691,7 +3784,7 @@ async def test_exited_returns_error(self, docker_backend, sample_deployment, mak """A terminated container should return ERROR.""" make_mock_container(status="exited", logs=b"Error occurred") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "ERROR" @@ -3700,7 +3793,7 @@ async def test_lost_returns_lost(self, docker_backend, sample_deployment, mock_d """A missing container (LOST) should return LOST.""" mock_docker_client.containers.get.side_effect = NotFound("Container not found") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "LOST" @@ -3719,7 +3812,7 @@ async def test_error_message_includes_docker_logs_command( with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "ERROR" assert "docker logs md-default-test-deployment" in status.status_message @@ -3736,7 +3829,7 @@ async def test_error_details_contain_container_logs(self, docker_backend, sample with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.error_details["error_stack"] is not None assert "NemotronHForCausalLM" in status.error_details["error_stack"] @@ -3750,7 +3843,7 @@ async def test_running_not_healthy_shows_restart_count_in_pending_message( with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "PENDING" assert "restarts: 3" in status.status_message @@ -3769,7 +3862,7 @@ async def test_running_not_healthy_exceeds_max_restarts_returns_error( with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "ERROR" assert "crash loop" in status.status_message @@ -3789,7 +3882,7 @@ async def test_running_not_healthy_below_max_restarts_returns_pending( with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "PENDING" assert "restarts: 3" in status.status_message @@ -3802,7 +3895,7 @@ async def test_restarting_state_exceeds_max_restarts_returns_error( make_mock_container(status="restarting", restart_count=7, logs=b"Segfault in model loading") docker_backend._backend_config.max_restart_count = 5 - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "ERROR" assert "crash loop" in status.status_message @@ -3819,7 +3912,7 @@ async def test_restarting_state_below_max_restarts_returns_pending( make_mock_container(status="restarting", restart_count=2) docker_backend._backend_config.max_restart_count = 5 - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "PENDING" assert "restart count: 2" in status.status_message @@ -3836,7 +3929,7 @@ async def test_crash_loop_takes_priority_over_pending_timeout( with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "ERROR" assert status.error_details["reason"] == "crash_loop" @@ -3849,7 +3942,7 @@ async def test_custom_max_restart_count(self, docker_backend, sample_deployment, with patch.object(docker_backend, "_probe_nim_health", new_callable=AsyncMock) as mock_probe: mock_probe.return_value = (False, "connection refused") - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "PENDING" assert "restarts: 10" in status.status_message @@ -3862,7 +3955,7 @@ async def test_crash_loop_error_includes_container_logs( make_mock_container(status="restarting", restart_count=5, logs=b"RuntimeError: CUDA out of memory") docker_backend._backend_config.max_restart_count = 5 - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "ERROR" assert "CUDA out of memory" in status.error_details["error_stack"] @@ -3943,7 +4036,9 @@ async def test_create_returns_pending_with_pulling_message( mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - status = await docker_backend.create_model_deployment(sample_deployment, sample_config) + status = await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) assert status.status == "PENDING" assert "pulling container image" in status.status_message.lower() @@ -3957,7 +4052,9 @@ async def test_creation_state_stored_after_create( mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) key = docker_backend._get_deployment_key(sample_deployment) assert key in docker_backend._reconciler._creation_states @@ -3973,7 +4070,9 @@ async def test_get_status_delegates_to_advance_creation( mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) key = docker_backend._get_deployment_key(sample_deployment) assert key in docker_backend._reconciler._creation_states @@ -3983,7 +4082,7 @@ async def test_get_status_delegates_to_advance_creation( if state.task and not state.task.done(): await state.task - status = await docker_backend.get_model_deployment_status(sample_deployment) + status = await docker_backend.get_model_deployment_status(ModelContext(model_deployment=sample_deployment)) assert status.status == "PENDING" @pytest.mark.asyncio @@ -3995,7 +4094,9 @@ async def test_image_pull_failure_returns_error( mock_docker_client.images.pull.side_effect = ImageNotFound("Image not found in registry") mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status = await drive_creation_to_completion(docker_backend, sample_deployment) @@ -4015,7 +4116,9 @@ async def test_creation_state_removed_after_completion( mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend, sample_deployment) key = docker_backend._get_deployment_key(sample_deployment) @@ -4033,7 +4136,9 @@ async def test_delete_during_creation_cancels_task( mock_docker_client.volumes.get.side_effect = None mock_docker_client.volumes.get.return_value = mock_volume - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) key = docker_backend._get_deployment_key(sample_deployment) assert key in docker_backend._reconciler._creation_states @@ -4051,7 +4156,9 @@ async def test_shutdown_cancels_all_creation_tasks( mock_docker_client.images.get.return_value = MagicMock() mock_docker_client.containers.list.return_value = [] - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) key = docker_backend._get_deployment_key(sample_deployment) assert key in docker_backend._reconciler._creation_states @@ -4084,7 +4191,11 @@ async def test_puller_stage_with_files_service_weights( _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) - await docker_backend.create_model_deployment(sample_deployment, sample_config, model_entity) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity + ) + ) key = docker_backend._get_deployment_key(sample_deployment) state = docker_backend._reconciler._creation_states[key] @@ -4117,7 +4228,11 @@ async def test_puller_running_stage_reports_downloading( mock_puller = _setup_puller_mock_for_polling(mock_docker_client, sample_deployment, exit_code=0) mock_puller.status = "running" - await docker_backend.create_model_deployment(sample_deployment, sample_config, model_entity) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity + ) + ) key = docker_backend._get_deployment_key(sample_deployment) # Advance through PULLING_NIM_IMAGE and PULLING_PULLER_IMAGE @@ -4157,7 +4272,9 @@ async def test_concurrent_deployments_not_blocked(self, docker_backend, sample_c deployments.append(d) for d in deployments: - await docker_backend.create_model_deployment(d, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=d, model_deployment_config=sample_config) + ) # All three should have creation states for d in deployments: @@ -4189,7 +4306,9 @@ async def spy_to_thread(func, *args, **kwargs): "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", side_effect=spy_to_thread, ): - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) key = docker_backend._get_deployment_key(sample_deployment) state = docker_backend._reconciler._creation_states[key] @@ -4222,7 +4341,9 @@ async def spy_to_thread(func, *args, **kwargs): "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", side_effect=spy_to_thread, ): - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) status = await drive_creation_to_completion(docker_backend, sample_deployment) assert status.status != "ERROR", f"Creation failed: {status.status_message}" @@ -4264,7 +4385,11 @@ async def spy_to_thread(func, *args, **kwargs): "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", side_effect=spy_to_thread, ): - await docker_backend.create_model_deployment(sample_deployment, sample_config, model_entity) + await docker_backend.create_model_deployment( + ModelContext( + model_deployment=sample_deployment, model_deployment_config=sample_config, model_entity=model_entity + ) + ) await drive_creation_to_completion(docker_backend, sample_deployment) pull_calls = [c for c in to_thread_calls if getattr(c[0], "__name__", "") == "pull_image_if_not_local"] @@ -4295,7 +4420,9 @@ async def spy_to_thread(func, *args, **kwargs): "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", side_effect=spy_to_thread, ): - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend, sample_deployment) container_calls = [c for c in to_thread_calls if getattr(c[0], "__name__", "") == "create_and_start_container"] @@ -4328,7 +4455,9 @@ async def spy_to_thread(func, *args, **kwargs): "nmp.core.models.controllers.backends.docker.creation_reconciler.asyncio.to_thread", side_effect=spy_to_thread, ): - await docker_backend.create_model_deployment(sample_deployment, sample_config) + await docker_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) await drive_creation_to_completion(docker_backend, sample_deployment) assert any("pull" in name.lower() for name in to_thread_funcs), ( diff --git a/services/core/models/tests/unit/controllers/test_k8s_nim_operator_backend.py b/services/core/models/tests/unit/controllers/test_k8s_nim_operator_backend.py index 0ce34c6845..369b6a578b 100644 --- a/services/core/models/tests/unit/controllers/test_k8s_nim_operator_backend.py +++ b/services/core/models/tests/unit/controllers/test_k8s_nim_operator_backend.py @@ -5,6 +5,7 @@ import contextlib from datetime import datetime, timedelta, timezone +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -17,9 +18,15 @@ from nmp.core.models.controllers.backends.common import deployment_elapsed_seconds, format_duration from nmp.core.models.controllers.backends.k8s_nim_operator import K8sNimOperatorServiceBackend from nmp.core.models.controllers.backends.k8s_nim_operator.config import K8sNimOperatorConfig +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.resource_deleter import ResourceDeleter +from nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector import StatusProjector +from nmp.core.models.controllers.context import ModelContext from pydantic import ValidationError _K8S_BACKEND_MODULE = "nmp.core.models.controllers.backends.k8s_nim_operator.backend" +_RECON_K8S_MODULE = "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.k8s" +_RECON_STATUS_MODULE = "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.status_projector" +_RECON_NIM_MODULE = "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator" # --------------------------------------------------------------------------- @@ -27,6 +34,17 @@ # --------------------------------------------------------------------------- +def _nim_config(): + """A minimal NIM-routing ModelDeploymentConfig-like object. + + ``config_engine`` returns the NIM engine for anything that isn't explicitly + ``vllm``/``generic``, so a bare mock routes status/create/update to the NIM + reconciler. The NIM status path only reads ``resource_name``, so the resolved + model fields are irrelevant here. + """ + return MagicMock() + + def _make_nimservice_mock(state: str, conditions: list | None = None): """Create a mock NIMService response dict.""" mock_resource = MagicMock() @@ -92,8 +110,8 @@ def _mock_pod_backend(k8s_backend, pod=None, *, pod_logs=""): mock_core_v1.read_namespaced_pod_log.return_value = pod_logs with ( - patch(f"{_K8S_BACKEND_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1), - patch(f"{_K8S_BACKEND_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1), + patch(f"{_RECON_STATUS_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1), + patch(f"{_RECON_STATUS_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1), ): yield mock_apps_v1, mock_core_v1 @@ -115,7 +133,6 @@ def mock_k8s_config(): patch(f"{_K8S_BACKEND_MODULE}.k8s_config.load_kube_config"), patch(f"{_K8S_BACKEND_MODULE}.k8s_client.ApiClient"), patch(f"{_K8S_BACKEND_MODULE}.DynamicClient"), - patch(f"{_K8S_BACKEND_MODULE}.K8sNimOperatorServiceBackend._validate_nim_operator_crds"), patch(f"{_K8S_BACKEND_MODULE}.os.path.exists", return_value=False), ): yield @@ -182,6 +199,58 @@ def k8s_backend(mock_nmp_sdk, mock_k8s_config): ) +def _sync_reconcilers(backend): + """Propagate the backend's (test-mocked) k8s state onto its reconcilers. + + Tests assign mock clients/config/namespace onto the backend *after* + construction (``backend._dynamic_client = MagicMock()`` etc.). Reconciliation + logic now lives on the two reconcilers, which captured their own clients at + ``init()`` time. This helper re-points the reconcilers at whatever the test + set on the backend so delegation exercises the test's mocks. Call it after + setting up the backend's ``_dynamic_client`` / ``_core_v1`` / ``_apps_v1`` / + ``_batch_v1`` / ``_backend_config`` / ``_k8s_namespace`` / ``_k8s_client``. + """ + nim = backend._nim_reconciler + k8s = backend._k8s_reconciler + status = getattr(backend, "_status_projector", None) + deleter = getattr(backend, "_resource_deleter", None) + namespace = backend._k8s_namespace + config = backend._backend_config + client = backend._k8s_client + + if nim is not None: + nim._k8s_namespace = namespace + nim._backend_config = config + if backend._dynamic_client is not None: + nim._dynamic_client = backend._dynamic_client + if k8s is not None: + k8s._k8s_namespace = namespace + k8s._backend_config = config + k8s._k8s_client = client + if getattr(backend, "_core_v1", None) is not None: + k8s._core_v1 = backend._core_v1 + if getattr(backend, "_apps_v1", None) is not None: + k8s._apps_v1 = backend._apps_v1 + if getattr(backend, "_batch_v1", None) is not None: + k8s._batch_v1 = backend._batch_v1 + if status is not None: + status._k8s_namespace = namespace + status._backend_config = config + status._k8s_client = client + if deleter is not None: + deleter._k8s_namespace = namespace + return backend + + +def _status_helper_reconciler(*, namespace="default", backend_config=None, k8s_client_=None): + """Build a StatusProjector exposing the shared status helpers for direct tests.""" + return StatusProjector( + k8s_client_=k8s_client_ if k8s_client_ is not None else MagicMock(), + backend_config=backend_config if backend_config is not None else K8sNimOperatorConfig(), + k8s_namespace=namespace, + ) + + @pytest.fixture def sample_deployment(): """Create a sample ModelDeployment for testing. @@ -222,12 +291,17 @@ async def test_k8s_backend_create_model_deployment(k8s_backend, sample_deploymen k8s_backend._dynamic_client.resources.get.return_value = mock_resource # Mock the compile_nimservice function to avoid validation issues - with patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.compile_nimservice") as mock_compile: + with patch( + "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" + ) as mock_compile: mock_nimservice = MagicMock() mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} mock_compile.return_value = mock_nimservice - status_update = await k8s_backend.create_model_deployment(sample_deployment, sample_config) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) # Verify compile_nimservice was called mock_compile.assert_called_once() @@ -255,12 +329,17 @@ async def test_k8s_backend_update_model_deployment(k8s_backend, sample_deploymen k8s_backend._dynamic_client.resources.get.return_value = mock_resource # Mock the compile_nimservice function to avoid validation issues - with patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.compile_nimservice") as mock_compile: + with patch( + "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" + ) as mock_compile: mock_nimservice = MagicMock() mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} mock_compile.return_value = mock_nimservice - status_update = await k8s_backend.update_model_deployment(sample_deployment, sample_config) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.update_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) # Verify compile_nimservice was called mock_compile.assert_called_once() @@ -281,7 +360,10 @@ async def test_k8s_backend_get_model_deployment_status(k8s_backend, sample_deplo k8s_backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("Ready") - status_update = await k8s_backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert status_update is not None assert status_update.status == "READY" @@ -289,6 +371,27 @@ async def test_k8s_backend_get_model_deployment_status(k8s_backend, sample_deplo assert status_update.host_url is not None +@pytest.mark.asyncio +async def test_k8s_backend_get_status_without_config_is_unknown(k8s_backend, sample_deployment): + """No config -> backend cannot determine the engine/state, returns UNKNOWN. + + The controller retries on the next poll (which normally has a config) and + escalates to ERROR after its retry budget; the backend does not probe. + """ + k8s_backend._dynamic_client = MagicMock() + k8s_backend._k8s_namespace = "default" + k8s_backend._backend_config = K8sNimOperatorConfig() + _sync_reconcilers(k8s_backend) + + status_update = await k8s_backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=None) + ) + + assert status_update.status == "UNKNOWN" + # No reconciler/cluster lookups happen without a config. + k8s_backend._dynamic_client.resources.get.assert_not_called() + + @pytest.mark.asyncio async def test_k8s_backend_get_status_nimservice_not_found(k8s_backend, sample_deployment): """Test getting status when NIMService doesn't exist yet - should keep current status.""" @@ -300,7 +403,10 @@ async def test_k8s_backend_get_status_nimservice_not_found(k8s_backend, sample_d mock_resource.get.side_effect = k8s_dynamic_exceptions.NotFoundError(MagicMock()) k8s_backend._dynamic_client.resources.get.return_value = mock_resource - status_update = await k8s_backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert status_update is not None assert status_update.status == "LOST" @@ -318,13 +424,16 @@ async def test_k8s_backend_get_status_nimservice_not_ready(k8s_backend, sample_d k8s_backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") with patch.object( - k8s_backend, - "_get_pod_status_from_deployment", + k8s_backend._status_projector, + "pod_status_from_deployment", return_value=DeploymentStatusUpdate( status="PENDING", status_message="Waiting for NIMService to become ready", host_url=None ), ): - status_update = await k8s_backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert status_update is not None assert status_update.status == "PENDING" @@ -346,7 +455,10 @@ async def test_k8s_backend_get_status_nimservice_crash_loop_backoff(k8s_backend, pod = _make_pod(restart_count=5, waiting_reason="CrashLoopBackOff") with _mock_pod_backend(k8s_backend, pod=pod, pod_logs="ERROR: model failed to load"): - status_update = await k8s_backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert status_update is not None assert status_update.status == "ERROR" @@ -371,7 +483,10 @@ async def test_k8s_backend_get_status_nimservice_pod_restarts_below_threshold(k8 pod = _make_pod(restart_count=2) with _mock_pod_backend(k8s_backend, pod=pod): - status_update = await k8s_backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert status_update is not None assert status_update.status == "PENDING" @@ -392,7 +507,10 @@ async def test_k8s_backend_get_status_nimservice_pod_running_after_restarts(k8s_ pod = _make_pod(restart_count=5) # No waiting_reason → running with _mock_pod_backend(k8s_backend, pod=pod): - status_update = await k8s_backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert status_update is not None assert status_update.status == "PENDING" @@ -412,6 +530,7 @@ async def test_k8s_backend_delete_model_deployment(k8s_backend, sample_deploymen mock_resource = MagicMock() k8s_backend._dynamic_client.resources.get.return_value = mock_resource + _sync_reconcilers(k8s_backend) status_update = await k8s_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) # Verify status update returned @@ -433,6 +552,7 @@ async def test_k8s_backend_delete_model_deployment_with_secret(k8s_backend, samp mock_resource = MagicMock() k8s_backend._dynamic_client.resources.get.return_value = mock_resource + _sync_reconcilers(k8s_backend) status_update = await k8s_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) # Verify status update returned @@ -453,6 +573,7 @@ async def test_k8s_backend_delete_model_deployment_without_secret(k8s_backend, s mock_resource = MagicMock() k8s_backend._dynamic_client.resources.get.return_value = mock_resource + _sync_reconcilers(k8s_backend) status_update = await k8s_backend.delete_model_deployment(sample_deployment.workspace, sample_deployment.name) # Verify status update returned @@ -460,6 +581,116 @@ async def test_k8s_backend_delete_model_deployment_without_secret(k8s_backend, s assert status_update.status == "DELETED" +@pytest.mark.asyncio +async def test_delete_attempts_all_resource_types_and_tolerates_404(k8s_backend, sample_deployment): + """Delete attempts CRs + raw vLLM objects by name; 404s are success -> DELETED.""" + k8s_backend._k8s_namespace = "default" + k8s_backend._dynamic_client = MagicMock() + k8s_backend._core_v1 = MagicMock() + k8s_backend._apps_v1 = MagicMock() + k8s_backend._batch_v1 = MagicMock() + cr_api = MagicMock() + cr_api.delete.side_effect = k8s_dynamic_exceptions.NotFoundError(MagicMock(status=404)) + k8s_backend._dynamic_client.resources.get.return_value = cr_api + notfound = k8s_client.exceptions.ApiException(status=404) + k8s_backend._apps_v1.delete_namespaced_deployment.side_effect = notfound + k8s_backend._core_v1.delete_namespaced_service.side_effect = notfound + k8s_backend._batch_v1.delete_namespaced_job.side_effect = notfound + k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.side_effect = notfound + + _sync_reconcilers(k8s_backend) + result = await k8s_backend.delete_model_deployment("default", "qwen") + + assert result.status == "DELETED" + assert k8s_backend._dynamic_client.resources.get.call_count == 2 + k8s_backend._apps_v1.delete_namespaced_deployment.assert_called_once() + k8s_backend._core_v1.delete_namespaced_service.assert_called_once() + k8s_backend._batch_v1.delete_namespaced_job.assert_called_once() + k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.assert_called_once() + + +@pytest.mark.asyncio +async def test_delete_real_failure_surfaces_error_but_attempts_all(k8s_backend, sample_deployment): + """A non-404 delete failure -> ERROR (not DELETED), and other deletes still run.""" + k8s_backend._k8s_namespace = "default" + k8s_backend._dynamic_client = MagicMock() + k8s_backend._core_v1 = MagicMock() + k8s_backend._apps_v1 = MagicMock() + k8s_backend._batch_v1 = MagicMock() + cr_api = MagicMock() + cr_api.delete.side_effect = k8s_dynamic_exceptions.NotFoundError(MagicMock(status=404)) + k8s_backend._dynamic_client.resources.get.return_value = cr_api + k8s_backend._apps_v1.delete_namespaced_deployment.side_effect = k8s_client.exceptions.ApiException(status=500) + notfound = k8s_client.exceptions.ApiException(status=404) + k8s_backend._core_v1.delete_namespaced_service.side_effect = notfound + k8s_backend._batch_v1.delete_namespaced_job.side_effect = notfound + k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.side_effect = notfound + + _sync_reconcilers(k8s_backend) + result = await k8s_backend.delete_model_deployment("default", "qwen") + + assert result.status == "ERROR" + k8s_backend._core_v1.delete_namespaced_service.assert_called_once() + k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.assert_called_once() + + +@pytest.mark.asyncio +async def test_delete_forbidden_cr_does_not_block_vllm_cleanup(k8s_backend, sample_deployment): + """A 403 deleting a NIMService still lets the raw vLLM objects be deleted (and surfaces ERROR).""" + k8s_backend._k8s_namespace = "default" + k8s_backend._dynamic_client = MagicMock() + k8s_backend._core_v1 = MagicMock() + k8s_backend._apps_v1 = MagicMock() + k8s_backend._batch_v1 = MagicMock() + cr_api = MagicMock() + cr_api.delete.side_effect = k8s_dynamic_exceptions.ForbiddenError(MagicMock(status=403)) + k8s_backend._dynamic_client.resources.get.return_value = cr_api + + _sync_reconcilers(k8s_backend) + result = await k8s_backend.delete_model_deployment("default", "qwen") + + assert result.status == "ERROR" + assert "forbidden" in result.status_message.lower() + k8s_backend._apps_v1.delete_namespaced_deployment.assert_called_once() + k8s_backend._core_v1.delete_namespaced_persistent_volume_claim.assert_called_once() + + +def test_delete_one_404_is_success(): + """A typed 404 (object absent) is treated as success -> returns None.""" + deleter = ResourceDeleter(k8s_namespace="default") + delete_fn = MagicMock(side_effect=k8s_client.exceptions.ApiException(status=404)) + assert deleter.delete_one(delete_fn, "PVC", "obj") is None + + +def test_delete_one_dynamic_notfound_is_success(): + """A dynamic NotFoundError is treated as success -> returns None.""" + deleter = ResourceDeleter(k8s_namespace="default") + delete_fn = MagicMock(side_effect=k8s_dynamic_exceptions.NotFoundError(MagicMock(status=404))) + assert deleter.delete_one(delete_fn, "PVC", "obj") is None + + +def test_delete_one_forbidden_is_classified_not_raised(): + """A 403 is classified and returned as an error string, not raised.""" + deleter = ResourceDeleter(k8s_namespace="default") + delete_fn = MagicMock(side_effect=k8s_client.exceptions.ApiException(status=403)) + err = deleter.delete_one(delete_fn, "PVC", "obj") + assert err is not None + assert "forbidden" in err.lower() + + +def test_delete_one_unexpected_exception_is_classified_not_raised(): + """A non-API/transport error must be classified and returned, never raised. + + Guards the aggregation contract: the caller's per-resource delete loop must + continue (and surface the failure) rather than abort cleanup partway. + """ + deleter = ResourceDeleter(k8s_namespace="default") + delete_fn = MagicMock(side_effect=ConnectionError("connection reset")) + err = deleter.delete_one(delete_fn, "Deployment", "obj") + assert err is not None + assert "error deleting Deployment obj" in err + + def test_k8s_backend_initialization(mock_nmp_sdk, mock_k8s_config): """Test K8s NIM Operator backend initializes correctly with custom namespace config.""" config = {"namespace": "nim-system"} @@ -510,12 +741,17 @@ async def test_k8s_backend_create_when_nimservice_already_exists(k8s_backend, sa k8s_backend._dynamic_client.resources.get.return_value = mock_resource # Mock the compile_nimservice function - with patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.compile_nimservice") as mock_compile: + with patch( + "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" + ) as mock_compile: mock_nimservice = MagicMock() mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} mock_compile.return_value = mock_nimservice - status_update = await k8s_backend.create_model_deployment(sample_deployment, sample_config) + _sync_reconcilers(k8s_backend) + status_update = await k8s_backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=sample_config) + ) # Verify status update returned PENDING (not ERROR) assert status_update is not None @@ -600,13 +836,20 @@ def get_resource_side_effect(api_version, kind): "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", return_value=platform_config, ): - with patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.compile_nimservice") as mock_compile: + with patch( + "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" + ) as mock_compile: mock_nimservice = MagicMock() mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} mock_compile.return_value = mock_nimservice + _sync_reconcilers(k8s_backend) status_update = await k8s_backend.create_model_deployment( - sample_deployment, sample_config, sft_model_entity + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=sample_config, + model_entity=sft_model_entity, + ) ) # Verify NIMCache was created @@ -696,13 +939,20 @@ def get_resource_side_effect(api_version, kind): "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", return_value=platform_config, ): - with patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.compile_nimservice") as mock_compile: + with patch( + "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" + ) as mock_compile: mock_nimservice = MagicMock() mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} mock_compile.return_value = mock_nimservice + _sync_reconcilers(k8s_backend) status_update = await k8s_backend.create_model_deployment( - sample_deployment, sample_config, files_service_model_entity + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=sample_config, + model_entity=files_service_model_entity, + ) ) # Verify NIMCache was created for FILES_SERVICE @@ -759,13 +1009,20 @@ async def test_create_model_deployment_without_sft_model(k8s_backend, sample_dep k8s_backend._dynamic_client.resources.get.return_value = mock_nimservice_resource # Mock the compile_nimservice function - with patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.compile_nimservice") as mock_compile: + with patch( + "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" + ) as mock_compile: mock_nimservice = MagicMock() mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} mock_compile.return_value = mock_nimservice + _sync_reconcilers(k8s_backend) status_update = await k8s_backend.create_model_deployment( - sample_deployment, sample_config, non_sft_model_entity + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=sample_config, + model_entity=non_sft_model_entity, + ) ) # Verify compile_nimservice was called with nimcache_name=None @@ -843,7 +1100,9 @@ def get_resource(api_version, kind): k8s_backend._dynamic_client.resources.get.side_effect = get_resource # Mock the compile_nimservice function - with patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.compile_nimservice") as mock_compile: + with patch( + "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" + ) as mock_compile: mock_nimservice = MagicMock() mock_nimservice.model_dump.return_value = { "apiVersion": "apps.nvidia.com/v1alpha1", @@ -851,8 +1110,13 @@ def get_resource(api_version, kind): } mock_compile.return_value = mock_nimservice + _sync_reconcilers(k8s_backend) status_update = await k8s_backend.create_model_deployment( - sample_deployment, sample_config, sft_model_entity + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=sample_config, + model_entity=sft_model_entity, + ) ) # Verify NIMCache was created @@ -944,13 +1208,20 @@ def get_resource_side_effect(api_version, kind): "nmp.core.models.controllers.backends.k8s_nim_operator.nimservice_compiler.get_platform_config", return_value=platform_config, ): - with patch("nmp.core.models.controllers.backends.k8s_nim_operator.backend.compile_nimservice") as mock_compile: + with patch( + "nmp.core.models.controllers.backends.k8s_nim_operator.reconcilers.nim_operator.compile_nimservice" + ) as mock_compile: mock_nimservice = MagicMock() mock_nimservice.model_dump.return_value = {"apiVersion": "apps.nvidia.com/v1alpha1", "kind": "NIMService"} mock_compile.return_value = mock_nimservice + _sync_reconcilers(k8s_backend) status_update = await k8s_backend.create_model_deployment( - sample_deployment, sample_config, mismatched_model_entity + ModelContext( + model_deployment=sample_deployment, + model_deployment_config=sample_config, + model_entity=mismatched_model_entity, + ) ) mock_nimcache_resource.create.assert_called_once() @@ -1017,22 +1288,22 @@ def test_format_duration(self, seconds, expected): class TestWithRestartInfo: - """Tests for K8sNimOperatorServiceBackend._with_restart_info.""" + """Tests for StatusProjector._with_restart_info.""" def test_no_restarts(self): - assert K8sNimOperatorServiceBackend._with_restart_info("some status", 0) == "some status" + assert StatusProjector._with_restart_info("some status", 0) == "some status" def test_with_restarts(self): - assert K8sNimOperatorServiceBackend._with_restart_info("some status", 3) == "some status, restarts: 3" + assert StatusProjector._with_restart_info("some status", 3) == "some status, restarts: 3" class TestGetPodRestartCount: - """Tests for K8sNimOperatorServiceBackend._get_pod_restart_count.""" + """Tests for StatusProjector._get_pod_restart_count.""" def test_no_container_statuses(self): pod = MagicMock() pod.status.container_statuses = None - assert K8sNimOperatorServiceBackend._get_pod_restart_count(pod) == 0 + assert StatusProjector._get_pod_restart_count(pod) == 0 def test_multiple_containers_returns_max(self): pod = MagicMock() @@ -1041,7 +1312,7 @@ def test_multiple_containers_returns_max(self): cs2 = MagicMock() cs2.restart_count = 7 pod.status.container_statuses = [cs1, cs2] - assert K8sNimOperatorServiceBackend._get_pod_restart_count(pod) == 7 + assert StatusProjector._get_pod_restart_count(pod) == 7 class TestDeploymentElapsedSeconds: @@ -1089,11 +1360,14 @@ async def test_pending_within_timeout_stays_pending(self, backend_with_short_tim backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") with patch.object( - backend, - "_get_pod_status_from_deployment", + backend._status_projector, + "pod_status_from_deployment", return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), ): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "PENDING" # No elapsed/timeout in message (stable message to avoid new history entry every poll) @@ -1107,12 +1381,15 @@ async def test_pending_beyond_timeout_transitions_to_error(self, backend_with_sh backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") with patch.object( - backend, - "_get_pod_status_from_deployment", + backend._status_projector, + "pod_status_from_deployment", return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), ): with _mock_pod_backend(backend, pod_logs="timeout error log"): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "ERROR" assert "timed out" in result.status_message assert "kubectl logs" in result.status_message @@ -1127,7 +1404,10 @@ async def test_ready_not_affected_by_elapsed_time(self, backend_with_short_timeo backend._dynamic_client = MagicMock() backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("Ready") - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "READY" @pytest.mark.asyncio @@ -1142,7 +1422,10 @@ async def test_terminal_states_not_affected_by_timeout( backend._dynamic_client = MagicMock() backend._dynamic_client.resources.get.return_value = _make_nimservice_mock(nim_state) - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status != "PENDING" @pytest.mark.asyncio @@ -1156,12 +1439,15 @@ async def test_pending_timeout_at_exact_boundary(self, backend_with_short_timeou backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") with patch.object( - backend, - "_get_pod_status_from_deployment", + backend._status_projector, + "pod_status_from_deployment", return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), ): with _mock_pod_backend(backend): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "ERROR" assert result.error_details["reason"] == "pending_timeout" @@ -1176,28 +1462,27 @@ class TestCrashLoopDetection: @pytest.fixture def backend(self, mock_nmp_sdk, mock_k8s_config): - b = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - b._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=7200, max_restart_count=5) - b._k8s_namespace = "test-ns" - b._k8s_client = MagicMock() - return b + return _status_helper_reconciler( + namespace="test-ns", + backend_config=K8sNimOperatorConfig(pending_timeout_seconds=7200, max_restart_count=5), + ) def test_no_container_statuses(self, backend): pod = MagicMock() pod.metadata.name = "pod-1" pod.status.phase = "Pending" pod.status.container_statuses = None - assert backend._check_crash_loop(pod, "res-1") is None + assert backend.check_crash_loop(pod, "res-1") is None def test_below_threshold_no_error(self, backend): pod = _make_pod(restart_count=4, waiting_reason="CrashLoopBackOff") - assert backend._check_crash_loop(pod, "res-1") is None + assert backend.check_crash_loop(pod, "res-1") is None def test_at_threshold_with_waiting_returns_error(self, backend): pod = _make_pod(restart_count=5, waiting_reason="CrashLoopBackOff") with _mock_pod_backend(backend, pod_logs="crash log"): - result = backend._check_crash_loop(pod, "res-1") + result = backend.check_crash_loop(pod, "res-1") assert result is not None assert result.status == "ERROR" assert result.error_details["reason"] == "crash_loop" @@ -1206,7 +1491,7 @@ def test_at_threshold_with_waiting_returns_error(self, backend): def test_above_threshold_without_waiting_returns_none(self, backend): pod = _make_pod(restart_count=10) # running, not waiting - result = backend._check_crash_loop(pod, "res-1") + result = backend.check_crash_loop(pod, "res-1") assert result is None @pytest.mark.parametrize("max_restarts", [1, 3, 10]) @@ -1214,20 +1499,20 @@ def test_configurable_threshold(self, backend, max_restarts): backend._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=7200, max_restart_count=max_restarts) pod = _make_pod(restart_count=max_restarts, waiting_reason="CrashLoopBackOff") with _mock_pod_backend(backend, pod_logs=""): - result = backend._check_crash_loop(pod, "res-1") + result = backend.check_crash_loop(pod, "res-1") assert result is not None assert result.status == "ERROR" def test_crash_loop_error_includes_kubectl_command(self, backend): pod = _make_pod(name="my-pod-xyz", restart_count=5, waiting_reason="CrashLoopBackOff") with _mock_pod_backend(backend, pod_logs="some logs"): - result = backend._check_crash_loop(pod, "res-1") + result = backend.check_crash_loop(pod, "res-1") assert "kubectl logs -n test-ns my-pod-xyz" in result.status_message def test_crash_loop_error_includes_pod_logs_in_details(self, backend): pod = _make_pod(restart_count=5, waiting_reason="CrashLoopBackOff") with _mock_pod_backend(backend, pod_logs="RuntimeError: CUDA OOM"): - result = backend._check_crash_loop(pod, "res-1") + result = backend.check_crash_loop(pod, "res-1") assert result.error_details["error_stack"] == "RuntimeError: CUDA OOM" @@ -1241,15 +1526,14 @@ class TestPendingTimeoutErrorMessage: @pytest.fixture def backend(self, mock_nmp_sdk, mock_k8s_config): - b = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - b._backend_config = K8sNimOperatorConfig(pending_timeout_seconds=120, max_restart_count=5) - b._k8s_namespace = "my-ns" - b._k8s_client = MagicMock() - return b + return _status_helper_reconciler( + namespace="my-ns", + backend_config=K8sNimOperatorConfig(pending_timeout_seconds=120, max_restart_count=5), + ) def test_error_message_with_pod_name(self, backend): with _mock_pod_backend(backend, pod_logs="error log tail"): - result = backend._build_pending_timeout_error("my-resource", 150.0, "my-pod-123") + result = backend.build_pending_timeout_error("my-resource", 150.0, "my-pod-123") assert result.status == "ERROR" assert "timed out" in result.status_message assert "kubectl logs -n my-ns my-pod-123" in result.status_message @@ -1258,18 +1542,18 @@ def test_error_message_with_pod_name(self, backend): assert result.error_details["reason"] == "pending_timeout" def test_error_message_without_pod_name(self, backend): - result = backend._build_pending_timeout_error("my-resource", 150.0, None) + result = backend.build_pending_timeout_error("my-resource", 150.0, None) assert "kubectl logs -n my-ns deployment/my-resource" in result.status_message assert "pod_name" not in result.error_details def test_error_details_contain_timing(self, backend): - result = backend._build_pending_timeout_error("res", 200.0, None) + result = backend.build_pending_timeout_error("res", 200.0, None) assert result.error_details["elapsed_seconds"] == 200 assert result.error_details["timeout_seconds"] == 120 def test_crash_loop_error_message(self, backend): with _mock_pod_backend(backend, pod_logs="segfault"): - result = backend._build_crash_loop_error("res", "pod-abc", 7) + result = backend.build_crash_loop_error("res", "pod-abc", 7) assert result.status == "ERROR" assert "crash loop" in result.status_message assert "7 container restarts" in result.status_message @@ -1303,12 +1587,15 @@ async def test_timeout_detected_after_restart(self, mock_nmp_sdk, mock_k8s_confi backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") with patch.object( - backend, - "_get_pod_status_from_deployment", + backend._status_projector, + "pod_status_from_deployment", return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), ): with _mock_pod_backend(backend): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "ERROR" assert result.error_details["reason"] == "pending_timeout" @@ -1324,11 +1611,14 @@ async def test_no_false_positive_after_restart(self, mock_nmp_sdk, mock_k8s_conf backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") with patch.object( - backend, - "_get_pod_status_from_deployment", + backend._status_projector, + "pod_status_from_deployment", return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), ): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "PENDING" # No elapsed/timeout in message (stable message to avoid new history entry every poll) @@ -1343,27 +1633,24 @@ class TestFetchPodLogs: @pytest.fixture def backend(self, mock_nmp_sdk, mock_k8s_config): - b = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - b._k8s_namespace = "default" - b._k8s_client = MagicMock() - return b + return _status_helper_reconciler(namespace="default") def test_returns_logs_normally(self, backend): with _mock_pod_backend(backend, pod_logs="log line 1\nlog line 2"): - result = backend._fetch_pod_logs("pod-1") + result = backend.fetch_pod_logs("pod-1") assert result == "log line 1\nlog line 2" def test_truncates_long_logs(self, backend): long_logs = "x" * 3000 with _mock_pod_backend(backend, pod_logs=long_logs): - result = backend._fetch_pod_logs("pod-1") + result = backend.fetch_pod_logs("pod-1") assert len(result) == 2048 def test_returns_empty_on_exception(self, backend): mock_core_v1 = MagicMock() mock_core_v1.read_namespaced_pod_log.side_effect = Exception("API error") - with patch(f"{_K8S_BACKEND_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1): - result = backend._fetch_pod_logs("pod-1") + with patch(f"{_RECON_STATUS_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1): + result = backend.fetch_pod_logs("pod-1") assert result == "" @@ -1391,11 +1678,14 @@ async def test_timing_info_appended_to_pending(self, backend, sample_deployment) backend._dynamic_client.resources.get.return_value = _make_nimservice_mock("NotReady") with patch.object( - backend, - "_get_pod_status_from_deployment", + backend._status_projector, + "pod_status_from_deployment", return_value=DeploymentStatusUpdate(status="PENDING", status_message="Waiting", host_url=None), ): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "PENDING" # No elapsed/timeout appended (stable message to avoid new history entry every poll) assert result.status_message == "Waiting" @@ -1409,7 +1699,10 @@ async def test_restart_info_in_pod_status(self, backend, sample_deployment): pod = _make_pod(restart_count=3, phase="Running") with _mock_pod_backend(backend, pod=pod): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "PENDING" assert "restarts: 3" in result.status_message @@ -1422,7 +1715,10 @@ async def test_no_restart_info_when_zero(self, backend, sample_deployment): pod = _make_pod(restart_count=0, phase="Running") with _mock_pod_backend(backend, pod=pod): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "PENDING" assert "restarts" not in result.status_message @@ -1437,15 +1733,12 @@ class TestFindPodName: @pytest.fixture def backend(self, mock_nmp_sdk, mock_k8s_config): - b = K8sNimOperatorServiceBackend(nmp_sdk=mock_nmp_sdk, config={}, huggingface_model_puller="img:tag") - b._k8s_namespace = "default" - b._k8s_client = MagicMock() - return b + return _status_helper_reconciler(namespace="default") def test_returns_pod_name(self, backend): pod = _make_pod(name="found-pod-xyz") with _mock_pod_backend(backend, pod=pod) as (mock_apps_v1, _): - result = backend._find_pod_name("resource-1") + result = backend.find_pod_name("resource-1") assert result == "found-pod-xyz" def test_returns_none_when_no_pods(self, backend): @@ -1459,18 +1752,18 @@ def test_returns_none_when_no_pods(self, backend): mock_core_v1.list_namespaced_pod.return_value = pods_list with ( - patch(f"{_K8S_BACKEND_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1), - patch(f"{_K8S_BACKEND_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1), + patch(f"{_RECON_STATUS_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1), + patch(f"{_RECON_STATUS_MODULE}.k8s_client.CoreV1Api", return_value=mock_core_v1), ): - result = backend._find_pod_name("resource-1") + result = backend.find_pod_name("resource-1") assert result is None def test_returns_none_on_api_exception(self, backend): mock_apps_v1 = MagicMock() mock_apps_v1.read_namespaced_deployment.side_effect = k8s_client.exceptions.ApiException(status=404) - with patch(f"{_K8S_BACKEND_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1): - result = backend._find_pod_name("resource-1") + with patch(f"{_RECON_STATUS_MODULE}.k8s_client.AppsV1Api", return_value=mock_apps_v1): + result = backend.find_pod_name("resource-1") assert result is None @@ -1496,7 +1789,10 @@ async def test_crash_loop_overrides_timeout(self, mock_nmp_sdk, mock_k8s_config, pod = _make_pod(restart_count=3, waiting_reason="CrashLoopBackOff") with _mock_pod_backend(backend, pod=pod, pod_logs="crash"): - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "ERROR" assert result.error_details["reason"] == "crash_loop" @@ -1521,7 +1817,10 @@ async def test_failed_nimservice_returns_error(self, mock_nmp_sdk, mock_k8s_conf mock_resource = _make_nimservice_mock("Failed", [{"type": "Failed", "message": "out of GPU memory"}]) backend._dynamic_client.resources.get.return_value = mock_resource - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "ERROR" assert "NIMService failed" in result.status_message @@ -1547,7 +1846,10 @@ async def test_lost_nimservice_not_treated_as_pending(self, mock_nmp_sdk, mock_k mock_resource.get.side_effect = k8s_dynamic_exceptions.NotFoundError(MagicMock()) backend._dynamic_client.resources.get.return_value = mock_resource - result = await backend.get_model_deployment_status(sample_deployment) + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=_nim_config()) + ) assert result.status == "LOST" @@ -1580,6 +1882,7 @@ async def test_list_managed_deployment_names_returns_workspace_name_from_labels( mock_nimservice_api.get.return_value = list_result backend._dynamic_client.resources.get.return_value = mock_nimservice_api + _sync_reconcilers(backend) names = await backend.list_managed_deployment_names() assert names == ["ws-a/dep1", "ws-b/dep2"] @@ -1598,6 +1901,7 @@ async def test_list_managed_deployment_names_empty_when_no_resources(mock_nmp_sd mock_nimservice_api.get.return_value = list_result backend._dynamic_client.resources.get.return_value = mock_nimservice_api + _sync_reconcilers(backend) names = await backend.list_managed_deployment_names() assert names == [] @@ -1609,8 +1913,366 @@ async def test_delete_model_deployment_by_id_calls_delete_resources(mock_nmp_sdk with patch(f"{_K8S_BACKEND_MODULE}.K8sNimOperatorServiceBackend._get_current_namespace", return_value="default"): backend = K8sNimOperatorServiceBackend(mock_nmp_sdk, {}, "pull-image") backend._k8s_namespace = "default" - with patch.object(backend, "_delete_resources_by_model_deployment_id") as mock_delete: + with patch.object(backend, "_delete_resources_by_model_deployment_id", new_callable=AsyncMock) as mock_delete: mock_delete.return_value = DeploymentStatusUpdate(status="DELETED", status_message="") result = await backend.delete_model_deployment("my-ws", "my-name") mock_delete.assert_called_once_with("my-ws", "my-name") assert result.status == "DELETED" + + +# =========================================================================== +# vLLM path (native Kubernetes objects, no operator) +# =========================================================================== + + +def _vllm_config(*, gpu: int = 1, lora_enabled: bool = False): + """A minimal vLLM ModelDeploymentConfig-like object for dispatch/compile.""" + return SimpleNamespace( + engine="vllm", + model_spec=SimpleNamespace( + model_type=None, + model_namespace="default", + model_name="qwen", + model_revision=None, + chat_template=None, + tool_call_config=None, + lora_enabled=lora_enabled, + ), + executor_config=SimpleNamespace( + gpu=gpu, + disk_size="50Gi", + image_name=None, + image_tag=None, + health_check_path=None, + additional_envs=None, + additional_args=[], + k8s_nim_operator_config=None, + override_config=None, + ), + ) + + +def _vllm_backend(k8s_backend): + """Wire a k8s_backend with mocked typed clients for the vLLM path.""" + k8s_backend._k8s_namespace = "nemo" + k8s_backend._backend_config = K8sNimOperatorConfig() + k8s_backend._k8s_client = MagicMock() + k8s_backend._core_v1 = MagicMock() + k8s_backend._apps_v1 = MagicMock() + k8s_backend._batch_v1 = MagicMock() + return k8s_backend + + +def _api_exception(status: int): + return k8s_client.exceptions.ApiException(status=status) + + +@pytest.mark.asyncio +async def test_vllm_create_emits_pvc_and_job_only(k8s_backend, sample_deployment): + """vLLM create (phase P0) emits the PVC + puller Job, not the Deployment/Service.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config(gpu=2) + + with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): + _sync_reconcilers(backend) + result = await backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) + ) + + assert result.status == "PENDING" + backend._core_v1.create_namespaced_persistent_volume_claim.assert_called_once() + backend._batch_v1.create_namespaced_job.assert_called_once() + # Deployment + Service are NOT created at P0. + backend._apps_v1.create_namespaced_deployment.assert_not_called() + backend._core_v1.create_namespaced_service.assert_not_called() + + # The puller Job requests the same GPU as the server (topology pin). + job = backend._batch_v1.create_namespaced_job.call_args.kwargs["body"] + assert job.spec.template.spec.containers[0].resources.requests["nvidia.com/gpu"] == "2" + + +@pytest.mark.asyncio +async def test_generic_engine_rejected_on_k8s(k8s_backend, sample_deployment): + """The generic engine is explicitly unsupported on the k8s backend.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config() + config.engine = "generic" + _sync_reconcilers(backend) + result = await backend.create_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) + ) + assert result.status == "ERROR" + assert "generic" in result.status_message.lower() + + +@pytest.mark.asyncio +async def test_vllm_status_job_running_is_pending(k8s_backend, sample_deployment): + """While the puller Job is running, status is PENDING.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config() + job = MagicMock() + job.status.failed = None + job.status.succeeded = None + backend._batch_v1.read_namespaced_job.return_value = job + # No serving Deployment yet (still in pull phase). + backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) + + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config) + ) + assert result.status == "PENDING" + assert "weights" in result.status_message.lower() + backend._apps_v1.create_namespaced_deployment.assert_not_called() + + +@pytest.mark.asyncio +async def test_vllm_status_job_failed_is_error(k8s_backend, sample_deployment): + """A failed puller Job surfaces as ERROR.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config() + job = MagicMock() + job.status.failed = 5 + job.status.succeeded = None + backend._batch_v1.read_namespaced_job.return_value = job + backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) + # No serving Deployment yet (failed during pull phase). + backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) + + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config) + ) + assert result.status == "ERROR" + assert result.error_details["reason"] == "weight_pull_failed" + + +@pytest.mark.asyncio +async def test_vllm_status_job_complete_creates_deployment(k8s_backend, sample_deployment): + """When the Job completes (phase P3), the Deployment + Service are created.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config(gpu=1) + + job = MagicMock() + job.status.failed = None + job.status.succeeded = 1 + backend._batch_v1.read_namespaced_job.return_value = job + # Deployment does not exist yet -> triggers P3 creation. + backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) + # After the puller Job is deleted, no puller pod remains (volume released). + backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) + created_dep = MagicMock() + created_dep.metadata.name = backend._get_resource_name(sample_deployment) + created_dep.metadata.uid = "dep-uid" + backend._apps_v1.create_namespaced_deployment.return_value = created_dep + + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config) + ) + + assert result.status == "PENDING" + # Puller Job deleted (release RWO volume) before the Deployment is created. + backend._batch_v1.delete_namespaced_job.assert_called_once() + backend._apps_v1.create_namespaced_deployment.assert_called_once() + backend._core_v1.create_namespaced_service.assert_called_once() + # ownerRef patched onto the PVC so it cascades with the Deployment (Job is gone). + backend._core_v1.patch_namespaced_persistent_volume_claim.assert_called_once() + backend._batch_v1.patch_namespaced_job.assert_not_called() + + +@pytest.mark.asyncio +async def test_vllm_status_p3_waits_for_puller_pod_to_release_volume(k8s_backend, sample_deployment): + """At P3, if the puller pod is still present, defer Deployment creation (RWO release).""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config(gpu=1) + + job = MagicMock() + job.status.failed = None + job.status.succeeded = 1 + backend._batch_v1.read_namespaced_job.return_value = job + backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) + # Puller pod still terminating -> volume not yet released. + backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[MagicMock()]) + + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config) + ) + + assert result.status == "PENDING" + backend._batch_v1.delete_namespaced_job.assert_called_once() + # Deployment is NOT created until the puller pod is gone. + backend._apps_v1.create_namespaced_deployment.assert_not_called() + + +@pytest.mark.asyncio +async def test_vllm_status_job_complete_with_lora_wires_sidecar(k8s_backend, sample_deployment): + """At P3 with LoRA enabled, the Deployment gets the cache-init + adapter sidecar.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config(gpu=1, lora_enabled=True) + + job = MagicMock() + job.status.failed = None + job.status.succeeded = 1 + backend._batch_v1.read_namespaced_job.return_value = job + backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) + backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) + created_dep = MagicMock() + created_dep.metadata.name = backend._get_resource_name(sample_deployment) + created_dep.metadata.uid = "dep-uid" + backend._apps_v1.create_namespaced_deployment.return_value = created_dep + + platform_cfg = MagicMock() + platform_cfg.image_pull_secrets = [] + platform_cfg.image_registry = "my-registry" + platform_cfg.image_tag = "local" + platform_cfg.to_shared_envvars.return_value = {"NMP_SHARED": "1"} + with patch(f"{_RECON_K8S_MODULE}.get_platform_config", return_value=platform_cfg): + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config) + ) + + assert result.status == "PENDING" + dep_obj = backend._apps_v1.create_namespaced_deployment.call_args.kwargs["body"] + pod = dep_obj.spec.template.spec + assert pod.init_containers[0].name == "lora-cache-init" + sidecar = next(ctr for ctr in pod.containers if ctr.name == "lora-sidecar") + env = {e.name: e.value for e in sidecar.env} + assert env["NIM_PEFT_SOURCE"] == "/scratch/loras" + assert env["VLLM_LORA_BASE_MODEL_OVERRIDE"] == "/model-store" + assert env["NMP_SHARED"] == "1" + + +@pytest.mark.asyncio +async def test_vllm_status_job_absent_pvc_present_resumes_p3_not_lost(k8s_backend, sample_deployment): + """Job deleted (RWO release) + PVC present + no Deployment -> resume P3, not LOST.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config(gpu=1) + + backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) + backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) # Job already deleted + # PVC still present -> we're mid-P3, not orphaned. + backend._core_v1.read_namespaced_persistent_volume_claim.return_value = MagicMock() + backend._core_v1.list_namespaced_pod.return_value = MagicMock(items=[]) + created_dep = MagicMock() + created_dep.metadata.name = backend._get_resource_name(sample_deployment) + created_dep.metadata.uid = "dep-uid" + backend._apps_v1.create_namespaced_deployment.return_value = created_dep + + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config) + ) + + assert result.status == "PENDING" + assert result.status != "LOST" + backend._apps_v1.create_namespaced_deployment.assert_called_once() + + +@pytest.mark.asyncio +async def test_vllm_status_job_and_pvc_absent_is_lost(k8s_backend, sample_deployment): + """Both Job and PVC gone + no Deployment -> genuine drift -> LOST.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config(gpu=1) + + backend._apps_v1.read_namespaced_deployment.side_effect = _api_exception(404) + backend._batch_v1.read_namespaced_job.side_effect = _api_exception(404) + backend._core_v1.read_namespaced_persistent_volume_claim.side_effect = _api_exception(404) + + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config) + ) + assert result.status == "LOST" + + +@pytest.mark.asyncio +async def test_vllm_status_deployment_ready_is_ready(k8s_backend, sample_deployment): + """A ready serving Deployment maps to READY + host_url.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config() + + job = MagicMock() + job.status.failed = None + job.status.succeeded = 1 + backend._batch_v1.read_namespaced_job.return_value = job + + dep = MagicMock() + dep.status.ready_replicas = 1 + backend._apps_v1.read_namespaced_deployment.return_value = dep + + _sync_reconcilers(backend) + result = await backend.get_model_deployment_status( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config) + ) + assert result.status == "READY" + assert result.host_url is not None + + +@pytest.mark.asyncio +async def test_vllm_update_unchanged_source_does_not_repull(k8s_backend, sample_deployment): + """Unchanged model source: no Job re-create, no resource deletion.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config() + + existing_job = MagicMock() + existing_job.metadata.labels = {"nmp.nvidia.com/engine": "vllm"} + existing_job.metadata.annotations = {"nmp.nvidia.com/model-source": "default/qwen"} + backend._batch_v1.read_namespaced_job.return_value = existing_job + + with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", None)): + with patch.object(backend._k8s_reconciler, "_delete_vllm_resources") as mock_delete: + _sync_reconcilers(backend) + result = await backend.update_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) + ) + + mock_delete.assert_not_called() + backend._batch_v1.create_namespaced_job.assert_not_called() + assert result.status == "PENDING" + + +@pytest.mark.asyncio +async def test_vllm_update_changed_source_repulls(k8s_backend, sample_deployment): + """Changed model source: delete resources and re-run the phased create.""" + backend = _vllm_backend(k8s_backend) + config = _vllm_config() + + existing_job = MagicMock() + existing_job.metadata.labels = {"nmp.nvidia.com/engine": "vllm"} + existing_job.metadata.annotations = {"nmp.nvidia.com/model-source": "default/old-model"} + backend._batch_v1.read_namespaced_job.return_value = existing_job + + with patch.object(backend, "_resolve_model_source", return_value=("default", "qwen", "v2")): + with patch.object(backend._k8s_reconciler, "_delete_vllm_resources") as mock_delete: + _sync_reconcilers(backend) + result = await backend.update_model_deployment( + ModelContext(model_deployment=sample_deployment, model_deployment_config=config, model_entity=None) + ) + + mock_delete.assert_called_once() + # Re-pull: a new PVC + Job are created. + backend._core_v1.create_namespaced_persistent_volume_claim.assert_called_once() + backend._batch_v1.create_namespaced_job.assert_called_once() + assert result.status == "PENDING" + + +@pytest.mark.asyncio +async def test_vllm_list_managed_unions_deployments(k8s_backend): + """list_managed_deployment_names unions NIMServices and raw vLLM Deployments.""" + backend = _vllm_backend(k8s_backend) + backend._dynamic_client = MagicMock() + backend._dynamic_client.resources.get.return_value.get.return_value = MagicMock(items=[]) + + dep = MagicMock() + dep.metadata.labels = { + "nmp.nvidia.com/deployment-workspace": "default", + "nmp.nvidia.com/deployment-name": "qwen", + } + backend._apps_v1.list_namespaced_deployment.return_value = MagicMock(items=[dep]) + + _sync_reconcilers(backend) + names = await backend.list_managed_deployment_names() + assert "default/qwen" in names